Analyses of learning via shared features experiments

# disable scientific notation
options(scipen = 999) 

# dependencies
library(tidyverse)
library(effsize)
library(foreign)
library(timesavers)
library(BayesFactor)
library(weights)
library(sjPlot)
library(nnet)
library(metafor)
library(car)
library(lsr)
library(knitr)
library(kableExtra)

Get data

data_exp_1 <- read_csv("../experiment 1/data/processed/data_for_analysis.csv")
data_exp_2 <- read_csv("../experiment 2/data/processed/data_for_analysis.csv")
data_exp_3 <- read_csv("../experiment 3/data/processed/data_for_analysis.csv")
data_exp_4 <- read_csv("../experiment 4/data/processed/data_for_analysis.csv")
data_exp_5 <- read_csv("../experiment 5/data/processed/data_for_analysis.csv")
data_exp_6 <- read_csv("../experiment 6/data/processed/data_for_analysis.csv")
data_exp_7 <- read_csv("../experiment 7/data/processed/data_for_analysis.csv")
data_exp_8 <- read_csv("../experiment 8/data/processed/data_for_analysis.csv")
data_exp_9 <- read_csv("../experiment 9/data/processed/data_for_analysis.csv")

# subsets of conditions
data_exp_8nri <- data_exp_8 %>%
  filter(Bag_Matching == "Shared_Feature_Bag_Location_Condition_1")

data_exp_8ri <- data_exp_8 %>%
  filter(Bag_Matching == "Shared_Feature_Bag_Random_Condition_3")

Analytic strategy

# analysis workflow. is applied witin all studies ---------------

# between groups comparisons analysis workflow ------
t_test_analyses <- function(data) {
  
  # summary stats
  summary_stats <- data %>%
    group_by(IV) %>%
    dplyr::summarize(mean = mean(DV),
                     sd = sd(DV)) %>%
    round_df(2)
  
  # t test
  t_test <- t.test(formula = DV ~ IV,
                   alternative = "two.sided",
                   paired = FALSE,
                   data = data)
  
  # effect size
  # formula based cohen's d seems to be invariant to ordering of the levels, can't control whether the direction 
  # cohens_d <- effsize::cohen.d(formula = DV ~ IV,
  #                              paired = FALSE,
  #                              data = data)
  cohens_d <- effsize::cohen.d(data$DV[data$IV == "CS1_Positive_CS2_Negative"],
                               data$DV[data$IV == "CS2_Positive_CS1_Negative"],
                               paired = FALSE)
  
  # printing
  t_test_p <- t_test$p.value[[1]]
  t_test_p_apa <- ifelse(t_test_p < 0.001, "< .001", 
                         ifelse(t_test_p < 0.01,
                                paste("= ", rd(t_test_p, 3), sep = ""),  # rd() rounds, converts to string, and removes the leading 0.
                                paste("= ", rd(t_test_p, 2), sep = "")))
  
  t_test_output <- paste("t(", format(round(t_test$parameter[[1]], 2), nsmall = 2), 
                         ") = ", format(round(t_test$statistic[[1]], 2), nsmall = 2), 
                         ", p ", t_test_p_apa, 
                         ", d = ", format(round(cohens_d$estimate[[1]], 2), nsmall = 2), 
                         ", 95% CI = [", format(round(cohens_d$conf.int[["lower"]], 2), nsmall = 2), ", ", 
                         format(round(cohens_d$conf.int[["upper"]], 2), nsmall = 2), "]", 
                         sep = "")
  
  # bayes factor t test
  t_test_bf <- ttestBF(formula = DV ~ IV,
                       data = data)
  
  bf10 <- round(exp(t_test_bf@bayesFactor$bf), 2)
  
  return(list(summary_stats = summary_stats,
              t_test = t_test,
              cohens_d = cohens_d,
              t_test_output = t_test_output,
              t_test_p = t_test_p,
              t_test_bf = t_test_bf,
              bf10 = bf10))
}

# multinominal analysis workflow ------
multinominal_analysis <- function(data) {
  
  # fit model
  fit <- multinom(DV ~ IV, 
                  data = data)
  
  # calculate p values
  z <- summary(fit)$coefficients/summary(fit)$standard.errors
  p <- (1 - pnorm(abs(z), 0, 1)) * 2
  
  pvalues <- p %>%
    as.data.frame() %>%
    rownames_to_column(var = "var") %>%
    
    rename(p = "IVCS2_Positive_CS1_Negative") %>%
    mutate(p = apa_p_value(p)) %>%
    select(p)
  
  # calc odds ratios and CIs
  coefficients <- as.data.frame(summary(fit)$coefficients) %>%
    rownames_to_column(var = "var") %>%
    rename(logOR = "IVCS2_Positive_CS1_Negative") %>%
    select(var, logOR)
  
  se <- as.data.frame(summary(fit)$standard.errors) %>%
    rownames_to_column(var = "var") %>%
    rename(logOR_se = "IVCS2_Positive_CS1_Negative") %>%
    select(logOR_se)
  
  # combine
  results <- cbind(coefficients, se) %>%
    mutate(OR = round(exp(logOR), 2),
           OR_ci_lwr = round(exp(logOR - logOR_se*1.96), 2),
           OR_ci_upr = round(exp(logOR + logOR_se*1.96), 2)) %>%
    cbind(pvalues) 
  
  return(results)
}

# meta analysis workflow ------
add_heterogeneity_metrics_to_forest <- function(fit) {
    bquote(paste("RE Model (", 
                 italic('I')^"2", " = ", .(formatC(format(round(fit$I2, 1), nsmall = 1))),
                 "%, ", italic('H')^"2", " = ", .(formatC(format(round(fit$H2, 1), nsmall = 1))), ")"))
}

meta_analysis_workflow <- function(data, 
                                   effect_size_label, 
                                   reference_line, 
                                   exp_estimates = FALSE,
                                   plot = TRUE) {
  require(timesavers)
  require(tidyverse)
  require(metafor)
  
  # fit random effects model 
  fitted_model <- 
    rma(yi = yi, 
        sei = sei,
        data = data,
        slab = Experiment)
  
  p_value <- apa_p_value(fitted_model$pval)
  
  # model predictions
  meta_analysis_results <-
    predict(fitted_model, digits = 5) %>%
    as.data.frame() %>%
    gather() %>%
    round_df(2) %>%
    dplyr::rename(metric = key,
                  estimate = value) %>%
    mutate(metric = dplyr::recode(metric,
                                  "pred" = paste0("Meta analysed ", effect_size_label),
                                  "ci.lb" = "95% CI lower",
                                  "ci.ub" = "95% CI upper",
                                  "cr.lb" = "95% CR lower",
                                  "cr.ub" = "95% CR upper"))
  
  # source_valence_conditionally exponentiate estimates if converting log odds to odds ratios
  if(exp_estimates == TRUE) {
    meta_analysis_results <- meta_analysis_results %>%
      mutate(estimate = round(exp(estimate), 2))
  }
  
  meta_analysis_results <- rbind(meta_analysis_results,
                                 data.frame(metric = "p",
                                            estimate = p_value))
  
  # summarize results
  meta_analysis_results_text <- 
    paste0("k = ", fitted_model$k, ", ", 
           effect_size_label, " = ", meta_analysis_results$estimate[1],
           # dynamic indexing of some values as different models return variables in different locations, but relative location is reliable
           ", 95% CI = [", meta_analysis_results$estimate[length(meta_analysis_results$estimate)-4],  
           ", ", meta_analysis_results$estimate[length(meta_analysis_results$estimate)-3], 
           "], 95% CR = [", meta_analysis_results$estimate[length(meta_analysis_results$estimate)-2], 
           ", ", meta_analysis_results$estimate[length(meta_analysis_results$estimate)-1],
           "], p ", meta_analysis_results$estimate[length(meta_analysis_results$estimate)])
  
  heterogeneity_test_results_text <- 
    paste0("Q(df = ",    fitted_model$k - 1, ") = ", round(fitted_model$QE, 2), 
           ", p ",     ifelse(fitted_model$pval < 0.001, "< .001", as.character(paste("=", round(fitted_model$pval, 3)))),
           ", tau^2 = ", round(fitted_model$tau2, 2), 
           ", I^2 = ",   round(fitted_model$I2, 2), "%",
           ", H^2 = ",   round(fitted_model$H2, 2))
  
  # forest plot
  if (exp_estimates == TRUE) {
    transformation <- exp
  } else { 
    transformation <- NULL
  }
  
  
  if (plot == TRUE) {
    forest_plot <- forest(fitted_model,
                          xlab = effect_size_label,
                          addcred = TRUE,
                          atransf = transformation,
                          refline = reference_line,
                          mlab = add_heterogeneity_metrics_to_forest(fitted_model))
  } else {
    forest_plot <- NULL
  }
  
  
  return(list(fitted_model = fitted_model, 
              meta_analysis_results = meta_analysis_results,
              meta_analysis_results_text = meta_analysis_results_text,
              heterogeneity_test_results_text = heterogeneity_test_results_text,
              forest_plot = forest_plot))
}


# robustness between original test and subset -----------
robust_test <- function(subset_test, original_test) {
  
  subset_p <- subset_test$estimate[subset_test$metric == "p"] %>%
    str_remove(., "=") %>%
    str_remove(., "<") %>%
    str_remove(., " ") %>%
    as.numeric()
  
  original_p <- original_test$estimate[original_test$metric == "p"] %>%
    str_remove(., "=") %>%
    str_remove(., "<") %>%
    str_remove(., " ") %>%
    as.numeric()
  
  return((subset_p >= 0.05 & original_p >= 0.05) | (subset_p < 0.05 & original_p < 0.05))
}


# moderator meta analysis workflow ------
multilevel_moderator_meta_analysis_workflow <- function(data, 
                                             effect_size_label = "Cohen's d") {
  require(timesavers)
  require(tidyverse)
  require(metafor)
  
  # fit random effects model 
  fitted_model <- 
    rma.mv(yi = yi, 
           V = sei^2,
           mods = ~ exclude,
           random =  ~as.factor(exclude) | Experiment,
           data = data,
           slab = paste(Experiment, ifelse(exclude == TRUE, "excluded", "retained"), sep = " - "))
  
  # summarize moderator results
  mod_meta_analysis_results_text <- 
    paste0("QM(df = 1) = ", round(fitted_model$QM, 2), 
           ", p ", ifelse(fitted_model$QMp < 0.001, "< .001", as.character(paste("=", round(fitted_model$QMp, 3)))),
           ", difference in excluded subset ", effect_size_label, " = ", round(fitted_model$b[2], 2), 
           ", 95% CI [", round(fitted_model$ci.lb[2], 2), ", ", round(fitted_model$ci.ub[2], 2), "]")
  
  return(list(fitted_model = fitted_model, 
              mod_meta_analysis_results_text = mod_meta_analysis_results_text))
  
}

Experiments

Experiment 1

Demographics

data_exp_1 %>% 
  dplyr::summarize(n = n(),
                   age_mean = mean(Age),
                   age_sd = sd(Age)) %>%
  round_df(2) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

n	age_mean	age_sd
103	33.06	8.43

data_exp_1 %>% 
  dplyr::count(Gender) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

Gender	n
Female	57
Male	46

H1: IAT

results_exp_1_h1 <- data_exp_1 %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed an automatic relative preference for CS1 over CS2 on the IAT (M = 0.37, SD = 0.46), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = -0.23, SD = 0.45), t(98.12) = 6.63, p < .001, d = 1.31, 95% CI = [0.88, 1.74], BF10 = 5714391.64.

H2: Self-report ratings

results_exp_1_h2 <- data_exp_1 %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed a relative preference on the self-report ratings for CS1 over CS2 (M = 3.33, SD = 4.60), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = -4.15, SD = 4.48), t(98.32) = 8.33, p < .001, d = 1.65, 95% CI = [1.20, 2.10], BF10 = 15170767375.77.

H3: Behavioural intentions

# set reference level to "Morag"
data_exp_1$intention <- relevel(as.factor(data_exp_1$intention), ref = "Morag")

results_exp_1_h3 <- data_exp_1 %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

Data from the behavioural intentions question were entered into a multinomial logistic regression with CSI as the reference category. Only results from the CS1-CS2 comparison are relevant to the hypothesis and will be reported here. Results demonstrated that participants responses of CS1 relative to CS2 differed between the two conditions, congruent with training, OR = 13.66, 95% CI = [3.56, 52.44], p = .0001.

Experiment 2

Demographics

data_exp_2 %>% 
  dplyr::summarize(n = n(),
                   age_mean = mean(Age),
                   age_sd = sd(Age)) %>%
  round_df(2) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

n	age_mean	age_sd
103	32.5	8.87

data_exp_2 %>% 
  dplyr::count(Gender) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

Gender	n
Female	58
Male	45

H1: IAT

results_exp_2_h1 <- data_exp_2 %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS2 eventually shared a color with positive words and CS1 shared a color with negative words, participants showed an automatic relative preference for CS2 over CS1 on the IAT (M = -0.26, SD = 0.54). whereas when CS2 eventually shared a color with negative words and CS1 shared a color with positive words, they demonstrated a relative preference for CS1 over CS2 (M = -0.12, SD = 0.60), t(100.85) = -1.18, p = .24, d = -0.23, 95% CI = [-0.62, 0.16], BF10 = 0.38.

Note that this result is in the opposite direction to all other experiments.

H2: Self-report ratings

results_exp_2_h2 <- data_exp_2 %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

Regardless of whether CS1 eventually shared a color with positive or negative words, participants showed a relative preference on the self-report ratings for CS2 over CS1 (CS1-positive condition: M = -2.80, SD = 5.33; CS1-negative condition: M = -1.58, SD = 6.03), t(100.98) = -1.09, p = .28, d = -0.21, 95% CI = [-0.61, 0.18], BF10 = 0.35.

Note that this null result is different to all other experiments.

H3: Behavioural intentions

# set reference level to "Morag"
data_exp_2$intention <- relevel(as.factor(data_exp_2$intention), ref = "Morag")

results_exp_2_h3 <- data_exp_2 %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

Data from the behavioural intentions question were entered into a multinomial logistic regression with CSI as the reference category. Only results from the CS1-CS2 comparison are relevant to the hypothesis and will be reported here. Results demonstrated that participants responses of CS1 relative to CS2 differed between the two conditions, but in the opposite direction to that predicted, OR = 0.22, 95% CI = [0.07, 0.66], p = .0073.

Note that this result is in the opposite direction to all other experiments.

Distribution of IAT and ratings

Experiment 3 was informed by the unexpected results of experiment 2, which prompted us to examine the distribution of scores on the IAT and self reports which suggested bi- and tri-modality. This is depicted below.

p1 <- 
  data_exp_2 %>%
  select(Participant, source_valence_condition, IAT_Score, EC_Effect_CS1_CS2_Difference_Score) %>%
  mutate(source_valence_condition = dplyr::recode(source_valence_condition,
                                                  CS1_Positive_CS2_Negative = "Morag positive & Struan negative",
                                                  CS2_Positive_CS1_Negative = "Morag negative & Struan positive")) %>%
  rename(`IAT D score` = IAT_Score,
         `Ratings difference score` = EC_Effect_CS1_CS2_Difference_Score) %>%
  gather(dv_modality, dv_score, c(`IAT D score`, `Ratings difference score`)) %>%
  ggplot(aes(dv_score)) +
  geom_density(adjust = 0.5) +
  facet_wrap(~dv_modality+source_valence_condition, scales = "free") +
  theme_classic() +
  theme(panel.grid.major = element_blank(),
        panel.grid.minor = element_blank(),
        strip.background = element_blank(),
        panel.border = element_blank()) +
  xlab("") +
  ylab("Probability density")

p1

Experiment 3

Demographics

data_exp_3 %>% 
  dplyr::summarize(n = n(),
                   age_mean = mean(Age),
                   age_sd = sd(Age)) %>%
  round_df(2) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

n	age_mean	age_sd
97	28.67	6.07

data_exp_3 %>% 
  dplyr::count(Gender) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

Gender	n
Female	58
Male	39

H1: IAT

results_exp_3_h1 <- data_exp_3 %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed an automatic relative preference for CS1 over CS2 on the IAT (M = 0.21, SD = 0.46), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = -0.15, SD = 0.59), t(93.42) = 3.29, p = .001, d = 0.66, 95% CI = [0.25, 1.08], BF10 = 20.10.

H2: Self-report ratings

results_exp_3_h2 <- data_exp_3 %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed a relative preference on the self-report ratings for CS1 over CS2 (M = 2.92, SD = 5.25), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = -2.85, SD = 5.02), t(92.94) = 5.52, p < .001, d = 1.13, 95% CI = [0.69, 1.56], BF10 = 45262.88.

H3: Behavioural intentions

# set reference level to "Morag"
data_exp_3$intention <- relevel(as.factor(data_exp_3$intention), ref = "Morag")

results_exp_3_h3 <- data_exp_3 %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

Data from the behavioural intentions question were entered into a multinomial logistic regression with CSI as the reference category. Only results from the CS1-CS2 comparison are relevant to the hypothesis and will be reported here. Results demonstrated that participants responses of CS1 relative to CS2 differed between the two conditions, congruent with training, OR = 6.94, 95% CI = [2.03, 23.77], p = .002.

Experiment 4

Demographics

data_exp_4 %>% 
  dplyr::summarize(n = n(),
                   age_mean = mean(Age),
                   age_sd = sd(Age)) %>%
  round_df(2) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

n	age_mean	age_sd
187	30.3	6.2

data_exp_4 %>% 
  dplyr::count(Gender) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

Gender	n
Female	94
Male	93

H1: IAT

results_exp_4_h1 <- data_exp_4 %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed an automatic relative preference for CS1 over CS2 on the IAT (M = 0.16, SD = 0.48), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = -0.18, SD = 0.46), t(184.00) = 5.02, p < .001, d = 0.73, 95% CI = [0.44, 1.03], BF10 = 11902.22.

H2: Self-report ratings

results_exp_4_h2 <- data_exp_4 %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed a relative preference on the self-report ratings for CS1 over CS2 (M = 3.57, SD = 4.99), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = -2.18, SD = 4.21), t(179.27) = 8.51, p < .001, d = 1.25, 95% CI = [0.93, 1.56], BF10 = 922634733581.12.

H3: Behavioural intentions

# set reference level to "Morag"
data_exp_4$intention <- relevel(as.factor(data_exp_4$intention), ref = "Morag")

results_exp_4_h3 <- data_exp_4 %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

Data from the behavioural intentions question were entered into a multinomial logistic regression with CSI as the reference category. Only results from the CS1-CS2 comparison are relevant to the hypothesis and will be reported here. Results demonstrated that participants responses of CS1 relative to CS2 differed between the two conditions, congruent with training, OR = 7.63, 95% CI = [3.11, 18.76], p < .0001.

Experiment 5

Demographics

data_exp_5 %>% 
  dplyr::summarize(n = n(),
                   age_mean = mean(Age),
                   age_sd = sd(Age)) %>%
  round_df(2) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

n	age_mean	age_sd
179	30.92	5.85

data_exp_5 %>% 
  dplyr::count(Gender) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

Gender	n
Female	95
Male	84

H1: IAT

results_exp_5_h1 <- data_exp_5 %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed an automatic relative preference for CS1 over CS2 on the IAT (M = 0.14, SD = 0.46), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = -0.12, SD = 0.46), t(168.75) = 3.79, p < .001, d = 0.57, 95% CI = [0.27, 0.87], BF10 = 109.05.

H2: Self-report ratings

results_exp_5_h2 <- data_exp_5 %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed a relative preference on the self-report ratings for CS1 over CS2 (M = 2.34, SD = 4.12), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = -2.38, SD = 4.09), t(169.77) = 7.66, p < .001, d = 1.15, 95% CI = [0.83, 1.47], BF10 = 5415593221.07.

H3: Behavioural intentions

# set reference level to "Morag"
data_exp_5$intention <- relevel(as.factor(data_exp_5$intention), ref = "Morag")

results_exp_5_h3 <- data_exp_5 %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

Data from the behavioural intentions question were entered into a multinomial logistic regression with CSI as the reference category. Only results from the CS1-CS2 comparison are relevant to the hypothesis and will be reported here. Results demonstrated that participants responses of CS1 relative to CS2 differed between the two conditions, congruent with training, OR = 5.00, 95% CI = [1.91, 13.06], p = .001.

Experiment 6

Demographics

data_exp_6 %>% 
  dplyr::summarize(n = n(),
                   age_mean = mean(Age),
                   age_sd = sd(Age)) %>%
  round_df(2) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

n	age_mean	age_sd
231	27.34	7.33

data_exp_6 %>% 
  dplyr::count(Gender) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

Gender	n
female	103
male	125
other	3

H1: IAT

results_exp_6_h1 <- data_exp_6 %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed an automatic relative preference for CS1 over CS2 on the IAT (M = 0.31, SD = 0.51), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = -0.18, SD = 0.52), t(227.66) = 7.25, p < .001, d = 0.96, 95% CI = [0.68, 1.23], BF10 = 1151492207.55.

H2: Self-report ratings

results_exp_6_h2 <- data_exp_6 %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed a relative preference on the self-report ratings for CS1 over CS2 (M = 3.61, SD = 4.72), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = -3.40, SD = 4.09), t(227.44) = 12.08, p < .001, d = 1.58, 95% CI = [1.29, 1.88], BF10 = 72613291696428081479680.00.

H3: Behavioural intentions

# set reference level to "Morag"
data_exp_6$intention <- relevel(as.factor(data_exp_6$intention), ref = "Morag")

results_exp_6_h3 <- data_exp_6 %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

Data from the behavioural intentions question were entered into a multinomial logistic regression with CSI as the reference category. Only results from the CS1-CS2 comparison are relevant to the hypothesis and will be reported here. Results demonstrated that participants responses of CS1 relative to CS2 differed between the two conditions, congruent with training, OR = 14.85, 95% CI = [6.98, 31.63], p < .0001.

Experiment 7

Demographics

data_exp_7 %>% 
  dplyr::summarize(n = n(),
                   age_mean = mean(Age, na.rm = TRUE),
                   age_sd = sd(Age, na.rm = TRUE)) %>%
  round_df(2) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

n	age_mean	age_sd
493	28.28	7.43

data_exp_7 %>% 
  dplyr::count(Gender) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

Gender	n
female	242
male	246
NA	5

H1: Evaluative Priming (Sequential BF)

N = 300

These analyses code each trial as being congruent or incongruent based on the valence of the prime and the valence of the source stimulus valence. Mean reaction time was calculated for each participant’s congruent and incongruent trials. These means for each trial type are then compared between participants. This represents the classical test of the EP effect.

data_300 <- data_exp_7 %>%
  slice(1:300) %>%
  select(ep_mean_rt_consistent, ep_mean_rt_inconsistent) %>%
  gather()

ttestBF(x = data_300$value[data_300$key == "ep_mean_rt_inconsistent"],
        y = data_300$value[data_300$key == "ep_mean_rt_consistent"],
        paired = TRUE)

## Bayes factor analysis
## --------------
## [1] Alt., r=0.707 : 1.41985 ±0%
## 
## Against denominator:
##   Null, mu = 0 
## ---
## Bayes factor type: BFoneSample, JZS

sample_300 <- 
  ttestBF(x = data_300$value[data_300$key == "ep_mean_rt_inconsistent"],
          y = data_300$value[data_300$key == "ep_mean_rt_consistent"],
          paired = TRUE,
          posterior = TRUE, 
          iterations = 5000)

data.frame(sample_300) %>%
  dplyr::summarize(mu_median = quantile(mu, prob = .500),
                   mu_lower = quantile(mu, prob = .025),
                   mu_upper = quantile(mu, prob = .975),
                   delta_median = quantile(delta, prob = .500),
                   delta_lower = quantile(delta, prob = .025),
                   delta_upper = quantile(delta, prob = .975)) %>%
  round(2) %>%
  gather(parameter, estimate)

##      parameter estimate
## 1    mu_median     4.20
## 2     mu_lower     0.95
## 3     mu_upper     7.50
## 4 delta_median     0.14
## 5  delta_lower     0.03
## 6  delta_upper     0.25

N = 400

data_400 <- data_exp_7 %>%
  slice(1:400) %>%
  select(ep_mean_rt_consistent, ep_mean_rt_inconsistent) %>%
  gather()

ttestBF(x = data_400$value[data_400$key == "ep_mean_rt_inconsistent"],
        y = data_400$value[data_400$key == "ep_mean_rt_consistent"],
        paired = TRUE)

## Bayes factor analysis
## --------------
## [1] Alt., r=0.707 : 0.7645194 ±0.01%
## 
## Against denominator:
##   Null, mu = 0 
## ---
## Bayes factor type: BFoneSample, JZS

sample_400 <- 
  ttestBF(x = data_400$value[data_400$key == "ep_mean_rt_inconsistent"],
          y = data_400$value[data_400$key == "ep_mean_rt_consistent"],
          paired = TRUE,
          posterior = TRUE, 
          iterations = 5000)

data.frame(sample_400) %>%
  dplyr::summarize(mu_median = quantile(mu, prob = .500),
                   mu_lower = quantile(mu, prob = .025),
                   mu_upper = quantile(mu, prob = .975),
                   delta_median = quantile(delta, prob = .500),
                   delta_lower = quantile(delta, prob = .025),
                   delta_upper = quantile(delta, prob = .975)) %>%
  round(2) %>%
  gather(parameter, estimate)

##      parameter estimate
## 1    mu_median     3.45
## 2     mu_lower     0.48
## 3     mu_upper     6.30
## 4 delta_median     0.12
## 5  delta_lower     0.02
## 6  delta_upper     0.21

N = 500

data_500 <- data_exp_7 %>%
  select(ep_mean_rt_consistent, ep_mean_rt_inconsistent) %>%
  gather()

ttestBF(x = data_500$value[data_500$key == "ep_mean_rt_inconsistent"],
        y = data_500$value[data_500$key == "ep_mean_rt_consistent"],
        paired = TRUE)

## Bayes factor analysis
## --------------
## [1] Alt., r=0.707 : 3.113847 ±0%
## 
## Against denominator:
##   Null, mu = 0 
## ---
## Bayes factor type: BFoneSample, JZS

sample_500 <- 
  ttestBF(x = data_500$value[data_500$key == "ep_mean_rt_inconsistent"],
          y = data_500$value[data_500$key == "ep_mean_rt_consistent"],
          paired = TRUE,
          posterior = TRUE, 
          iterations = 5000)

data.frame(sample_500) %>%
  dplyr::summarize(mu_median = quantile(mu, prob = .500),
                   mu_lower = quantile(mu, prob = .025),
                   mu_upper = quantile(mu, prob = .975),
                   delta_median = quantile(delta, prob = .500),
                   delta_lower = quantile(delta, prob = .025),
                   delta_upper = quantile(delta, prob = .975)) %>%
  round(2) %>%
  gather(parameter, estimate)

##      parameter estimate
## 1    mu_median     3.88
## 2     mu_lower     1.22
## 3     mu_upper     6.40
## 4 delta_median     0.13
## 5  delta_lower     0.04
## 6  delta_upper     0.21

Frequentist analysis of the final sample for reader familiarity

t.test(x = data_500$value[data_500$key == "ep_mean_rt_inconsistent"],
       y = data_500$value[data_500$key == "ep_mean_rt_consistent"],
       paired = TRUE)

## 
##  Paired t-test
## 
## data:  data_500$value[data_500$key == "ep_mean_rt_inconsistent"] and data_500$value[data_500$key == "ep_mean_rt_consistent"]
## t = 2.8909, df = 492, p-value = 0.004011
## alternative hypothesis: true difference in means is not equal to 0
## 95 percent confidence interval:
##  1.245710 6.531166
## sample estimates:
## mean of the differences 
##                3.888438

effsize::cohen.d(d = data_500$value,
                 f = fct_rev(data_500$key),
                 paired = TRUE)

## 
## Cohen's d
## 
## d estimate: 0.04949019 (negligible)
## 95 percent confidence interval:
##      lower      upper 
## 0.01587567 0.08310471

H2: Self-report ratings

Using the final analytic sample.

results_exp_7_h2 <- data_exp_7 %>%
  select(EC_Effect_CS1_CS2_Difference_Score, source_valence_condition) %>%
  na.omit() %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed a relative preference on the self-report ratings for CS1 over CS2 (M = 2.17, SD = 4.35), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = -2.66, SD = 4.02), t(479.98) = 12.72, p < .001, d = 1.15, 95% CI = [0.96, 1.35], BF10 = 80819093185460097270287433728.00.

H3: Behavioural intentions

Using the final analytic sample.

# set reference level to "Morag"
data_exp_7$intention <- relevel(as.factor(data_exp_7$intention), ref = "Morag")

results_exp_7_h3 <- data_exp_7 %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

Data from the behavioural intentions question were entered into a multinomial logistic regression with CSI as the reference category. Only results from the CS1-CS2 comparison are relevant to the hypothesis and will be reported here. Results demonstrated that participants responses of CS1 relative to CS2 differed between the two conditions, congruent with training, OR = 4.33, 95% CI = [2.77, 6.78], p < .0001.

Experiment 8

Demographics

data_exp_8 %>% 
  dplyr::summarize(n = n(),
                   age_mean = mean(Age),
                   age_sd = sd(Age)) %>%
  round_df(2) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

n	age_mean	age_sd
222	28.84	7.54

data_exp_8 %>% 
  dplyr::count(Gender) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

Gender	n
female	127
male	92
other	3

Comparing conditions

Pairwise follow up t-tests using Bonferroni-Holm corrections for multiple testing.

Results are not parsed into text automatically given the analysis workflow differs from other experiments, and this would take more coding effort than its worth.

H1: IAT

# anova
fit_exp_8_h1 <- aov(IAT_Score ~ Bag_Matching, 
                    contrasts = "contr.sum",
                    data = data_exp_8)

Anova(fit_exp_8_h1, type = "III")

## Anova Table (Type III tests)
## 
## Response: IAT_Score
##              Sum Sq  Df F value           Pr(>F)    
## (Intercept)   3.432   1  19.854 0.00001333305091 ***
## Bag_Matching  9.770   2  28.261 0.00000000001206 ***
## Residuals    37.854 219                             
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

# effect size
etaSquared(fit_exp_8_h1, type = 2, anova = FALSE )

##                 eta.sq eta.sq.part
## Bag_Matching 0.2051465   0.2051465

# follow up t tests
pairwise.t.test(data_exp_8$IAT_Score, 
                data_exp_8$Bag_Matching, 
                p.adj = "holm")

## 
##  Pairwise comparisons using t tests with pooled SD 
## 
## data:  data_exp_8$IAT_Score and data_exp_8$Bag_Matching 
## 
##                                         No_Shared_Feature_Condition_2
## Shared_Feature_Bag_Location_Condition_1 0.000000000041               
## Shared_Feature_Bag_Random_Condition_3   0.000000515586               
##                                         Shared_Feature_Bag_Location_Condition_1
## Shared_Feature_Bag_Location_Condition_1 -                                      
## Shared_Feature_Bag_Random_Condition_3   0.1                                    
## 
## P value adjustment method: holm

# means by condition
data_exp_8 %>%
  group_by(Bag_Matching) %>%
  dplyr::summarize(mean = mean(IAT_Score),
                   ci_lwr = mean - plotrix::std.error(IAT_Score)*1.96,
                   ci_upr = mean + plotrix::std.error(IAT_Score)*1.96) %>%
  round_df(2)

## # A tibble: 3 x 4
##   Bag_Matching                             mean ci_lwr ci_upr
##   <chr>                                   <dbl>  <dbl>  <dbl>
## 1 No_Shared_Feature_Condition_2           -0.2  -0.290  -0.12
## 2 Shared_Feature_Bag_Location_Condition_1  0.28  0.18    0.38
## 3 Shared_Feature_Bag_Random_Condition_3    0.16  0.06    0.26

H2: Self-report ratings

# anova
fit_exp_8_h2 <- aov(EC_Effect_CS1_CS2_Difference_Score ~ Bag_Matching, 
                    contrasts = "contr.sum",
                    data = data_exp_8)

Anova(fit_exp_8_h2, type = "III")

## Anova Table (Type III tests)
## 
## Response: EC_Effect_CS1_CS2_Difference_Score
##               Sum Sq  Df F value          Pr(>F)    
## (Intercept)   109.14   1  8.9765         0.00305 ** 
## Bag_Matching  649.36   2 26.7035 0.0000000000419 ***
## Residuals    2662.77 219                            
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

# effect size
etaSquared(fit_exp_8_h2, type = 2, anova = FALSE )

##                eta.sq eta.sq.part
## Bag_Matching 0.196056    0.196056

# follow up t tests
pairwise.t.test(data_exp_8$EC_Effect_CS1_CS2_Difference_Score, 
                data_exp_8$Bag_Matching, 
                p.adj = "holm")

## 
##  Pairwise comparisons using t tests with pooled SD 
## 
## data:  data_exp_8$EC_Effect_CS1_CS2_Difference_Score and data_exp_8$Bag_Matching 
## 
##                                         No_Shared_Feature_Condition_2
## Shared_Feature_Bag_Location_Condition_1 0.00000000062                
## Shared_Feature_Bag_Random_Condition_3   0.00000008789                
##                                         Shared_Feature_Bag_Location_Condition_1
## Shared_Feature_Bag_Location_Condition_1 -                                      
## Shared_Feature_Bag_Random_Condition_3   0.39                                   
## 
## P value adjustment method: holm

# means by condition
data_exp_8 %>%
  group_by(Bag_Matching) %>%
  dplyr::summarize(mean = mean(EC_Effect_CS1_CS2_Difference_Score),
                   ci_lwr = mean - plotrix::std.error(EC_Effect_CS1_CS2_Difference_Score)*1.96,
                   ci_upr = mean + plotrix::std.error(EC_Effect_CS1_CS2_Difference_Score)*1.96) %>%
  round_df(2)

## # A tibble: 3 x 4
##   Bag_Matching                             mean ci_lwr ci_upr
##   <chr>                                   <dbl>  <dbl>  <dbl>
## 1 No_Shared_Feature_Condition_2           -1.14  -1.78  -0.5 
## 2 Shared_Feature_Bag_Location_Condition_1  2.61   1.73   3.49
## 3 Shared_Feature_Bag_Random_Condition_3    2.1    1.21   3

Individual conditions for meta analysis

Standard non-random instructions condition

H1: IAT

results_exp_8nri_h1 <- data_exp_8nri %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed an automatic relative preference for CS1 over CS2 on the IAT (M = 0.44, SD = 0.32), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = 0.10, SD = 0.46), t(58.42) = 3.50, p < .001, d = 0.84, 95% CI = [0.35, 1.34], BF10 = 42.20.

H2: Self-report ratings

results_exp_8nri_h2 <- data_exp_8nri %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed a relative preference on the self-report ratings for CS1 over CS2 (M = 3.57, SD = 2.82), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = 1.57, SD = 4.41), t(55.35) = 2.26, p = .03, d = 0.55, 95% CI = [0.06, 1.03], BF10 = 2.25.

Random instructions condition for meta analysis

H1: IAT

results_exp_8ri_h1 <- data_exp_8ri %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed an automatic relative preference for CS1 over CS2 on the IAT (M = 0.45, SD = 0.40), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = 0.01, SD = 0.35), t(40.42) = 4.53, p < .001, d = 1.21, 95% CI = [0.66, 1.77], BF10 = 1255.98.

H2: Self-report ratings

results_exp_8ri_h2 <- data_exp_8ri %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

When CS1 eventually shared a color with positive words and CS2 shared a color with negative words, participants showed a relative preference on the self-report ratings for CS1 over CS2 (M = 3.53, SD = 3.90), whereas when CS1 eventually shared a color with negative words and CS2 shared a color with positive words, they demonstrated a relative preference for CS2 over CS1 (M = 1.35, SD = 3.46), t(40.33) = 2.26, p = .03, d = 0.60, 95% CI = [0.08, 1.13], BF10 = 2.52.

Experiment 9

Demographics

data_exp_9 %>% 
  dplyr::summarize(n = n(),
                   age_mean = mean(Age),
                   age_sd = sd(Age)) %>%
  round_df(2) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

n	age_mean	age_sd
89	31.53	7.35

data_exp_9 %>% 
  dplyr::count(Gender) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

Gender	n
female	47
male	42

H1: IAT

t.test(data_exp_9$IAT_Score, alternative = "two.sided")

## 
##  One Sample t-test
## 
## data:  data_exp_9$IAT_Score
## t = -3.8952, df = 88, p-value = 0.0001908
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -0.24189282 -0.07845558
## sample estimates:
##  mean of x 
## -0.1601742

data_exp_9 %>%
  dplyr::summarize(d = round(mean(IAT_Score)/sd(IAT_Score), 2))

## # A tibble: 1 x 1
##       d
##   <dbl>
## 1 -0.41

# ggplot(data_exp_9, aes(IAT_Score)) +
#   geom_density()

H2: Self-report ratings

t.test(data_exp_9$EC_Effect_CS1_CS2_Difference_Score, alternative = "two.sided")

## 
##  One Sample t-test
## 
## data:  data_exp_9$EC_Effect_CS1_CS2_Difference_Score
## t = -5.7116, df = 88, p-value = 0.0000001495
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
##  -2.101422 -1.016555
## sample estimates:
## mean of x 
## -1.558989

data_exp_9 %>%
  dplyr::summarize(d = round(mean(EC_Effect_CS1_CS2_Difference_Score)/sd(EC_Effect_CS1_CS2_Difference_Score), 2))

## # A tibble: 1 x 1
##       d
##   <dbl>
## 1 -0.61

# ggplot(data_exp_9, aes(EC_Effect_CS1_CS2_Difference_Score)) +
#   geom_density()

Meta analyses

Meta analyses of experiments 1-8 (nb not 9, which contained no manipulations but acted as an extension of experiment 8).

Random effects meta analyses were fitted using the metafor R package (Viechtbauer, 2010). The maximum liklihood estimator was used.

A separate meta analysis was fitted for each outcome variable (IAT, self reported ratings, behavioural intentions). Although multivariate meta analysis was a possible alternative (and is conducted in a separate markdown script), previous work has traditionally assessed these dependant variables separately, so we will too.

Sample sizes

Note that not all variables were collected in all subsets, so sample sizes for subsets do not equal the total sample size * the proportion meeting a given criteria.

total_n_across_studies <- 
  nrow(data_exp_1) + 
  nrow(data_exp_2) +
  nrow(data_exp_3) +
  nrow(data_exp_4) +
  nrow(data_exp_5) +
  nrow(data_exp_6) +
  nrow(data_exp_7) +
  nrow(data_exp_8)

data_combined <- 
  bind_rows(mutate(data_exp_1, experiment = 1),
            mutate(data_exp_2, experiment = 2),
            mutate(data_exp_3, experiment = 3),
            mutate(data_exp_4, experiment = 4),
            mutate(data_exp_5, experiment = 5),
            mutate(data_exp_6, experiment = 6),
            mutate(data_exp_7, experiment = 7),
            mutate(data_exp_8nri, experiment = 8),
            mutate(data_exp_8ri, experiment = 8)) %>%
  select(experiment, US_contingency_memory, color_valence_contingency_memory, 
         demand_IAT, demand_self_reports, hypothesis_awareness, reactance_IAT, reactance_self_reports) %>%
  # fudge to stop "unknown variable" warning from appearing
  mutate(DV = NA)

data_combined %>% 
  dplyr::summarize(full_sample_n = n()) %>%
  round_df(2) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

full_sample_n
1531

# percents by subset
temp_1 <- data_combined %>%
  count(US_contingency_memory) %>%
  spread(US_contingency_memory, n) %>%
  mutate(US_contingency_aware = `TRUE` / (`TRUE` + `FALSE`)) %>%
  select(US_contingency_aware)

temp_2 <- data_combined %>%
  count(color_valence_contingency_memory) %>%
  spread(color_valence_contingency_memory, n) %>%
  mutate(color_valence_contingency_aware = `TRUE` / (`TRUE` + `FALSE`)) %>%
  select(color_valence_contingency_aware)

temp_3 <- data_combined %>%
  rowwise() %>%
  mutate(demand = as.logical(max(c(demand_IAT, demand_self_reports)))) %>%
  ungroup() %>%
  count(demand) %>%
  spread(demand, n) %>%
  mutate(NOT_demand_compliant = `FALSE` / (`TRUE` + `FALSE`)) %>%
  select(NOT_demand_compliant)

temp_4a <- data_combined %>%
  count(reactance_IAT) %>%
  spread(reactance_IAT, n) %>%
  mutate(NOT_reactant_IAT = `FALSE` / (`TRUE` + `FALSE`)) %>%
  select(NOT_reactant_IAT)

temp_4b <- data_combined %>%
  count(reactance_self_reports) %>%
  spread(reactance_self_reports, n) %>%
  mutate(NOT_reactant_self_reports = `FALSE` / (`TRUE` + `FALSE`)) %>%
  select(NOT_reactant_self_reports)

temp_5 <- data_combined %>%
  count(hypothesis_awareness) %>%
  spread(hypothesis_awareness, n) %>%
  mutate(NOT_hypothesis_awareness = `FALSE` / (`TRUE` + `FALSE`)) %>%
  select(NOT_hypothesis_awareness)

bind_cols(temp_1,
          temp_2,
          temp_3,
          temp_4a,
          temp_4b,
          temp_5) %>%
  gather(subset, proportion) %>%
  round_df(3) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

subset	proportion
US_contingency_aware	0.775
color_valence_contingency_aware	0.596
NOT_demand_compliant	0.809
NOT_reactant_IAT	0.897
NOT_reactant_self_reports	0.858
NOT_hypothesis_awareness	0.267

# percents by subset
temp_1_by_exp <- data_combined %>%
  count(experiment, US_contingency_memory) %>%
  spread(US_contingency_memory, n) %>%
  mutate(US_contingency_aware = `TRUE` / (`TRUE` + `FALSE`)) %>%
  select(experiment, US_contingency_aware)

temp_2_by_exp <- data_combined %>%
  count(experiment, color_valence_contingency_memory) %>%
  spread(color_valence_contingency_memory, n) %>%
  mutate(color_valence_contingency_aware = `TRUE` / (`TRUE` + `FALSE`)) %>%
  select(experiment, color_valence_contingency_aware)

temp_3a_by_exp <- data_combined %>%
  count(experiment, demand_IAT) %>%
  spread(demand_IAT, n) %>%
  mutate(NOT_demand_IAT = `FALSE` / (`TRUE` + `FALSE`)) %>%
  select(experiment, NOT_demand_IAT)

temp_3b_by_exp <- data_combined %>%
  count(experiment, demand_self_reports) %>%
  spread(demand_self_reports, n) %>%
  mutate(NOT_demand_self_reports = `FALSE` / (`TRUE` + `FALSE`)) %>%
  select(experiment, NOT_demand_self_reports)

temp_4a_by_exp <- data_combined %>%
  count(experiment, reactance_IAT) %>%
  spread(reactance_IAT, n) %>%
  mutate(NOT_reactant_IAT = `FALSE` / (`TRUE` + `FALSE`)) %>%
  select(experiment, NOT_reactant_IAT)

temp_4b_by_exp <- data_combined %>%
  count(experiment, reactance_self_reports) %>%
  spread(reactance_self_reports, n) %>%
  mutate(NOT_reactant_self_reports = `FALSE` / (`TRUE` + `FALSE`)) %>%
  select(experiment, NOT_reactant_self_reports)

temp_5_by_exp <- data_combined %>%
  count(experiment, hypothesis_awareness) %>%
  spread(hypothesis_awareness, n) %>%
  mutate(NOT_hypothesis_awareness = `FALSE` / (`TRUE` + `FALSE`)) %>%
  select(experiment, NOT_hypothesis_awareness)

temp_1_by_exp %>%
  left_join(temp_2_by_exp, by = "experiment") %>%
  left_join(temp_3a_by_exp, by = "experiment") %>%
  left_join(temp_3b_by_exp, by = "experiment") %>%
  left_join(temp_4a_by_exp, by = "experiment") %>%
  left_join(temp_4b_by_exp, by = "experiment") %>%
  left_join(temp_5_by_exp, by = "experiment") %>%
  round_df(3) %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

experiment	US_contingency_aware	color_valence_contingency_aware	NOT_demand_IAT	NOT_demand_self_reports	NOT_reactant_IAT	NOT_reactant_self_reports	NOT_hypothesis_awareness
1	0.650	0.718	0.825	0.845	0.806	0.835	0.282
2	0.650	0.621	0.854	0.883	0.903	0.845	0.184
3	0.794	0.794	0.814	0.773	0.887	0.856	0.082
4	0.781	0.620	0.925	0.824	0.904	0.850	0.326
5	0.749	0.453	0.860	0.804	0.916	0.849	0.140
6	NA	NA	0.983	0.918	0.926	0.848	0.329
7	0.781	0.568	0.930	0.859	0.895	0.900	0.348
8	0.949	NA	0.986	0.870	0.891	0.775	0.145

Total N in all studies (for reporting in abstract) = 1615.

Meta analysed effect size

Notes on meta analysed effect size:

Credibility intervals refer to the interval of effect sizes likely to be observed in a future study on the basis of the width of estimation of the true effect size combined with variation in this true effect size (e.g., due to heterogeneity).

Notes on heterogeneity metrics:

• Q and its p value: A measure of weighted squared deviations. Depends on number of studies.
• tau^2: Population between-studies variance, ie estimated amount of total true heterogeneity in the effect. Is on the same scale as the meta analysed effect size, and so cannot be compared between meta analysis using different scales (e.g., Cohen’s d vs. Odds Ratios).
• I^2: Percentage of variation in the observed effects that is due to true heterogeneity (as quantified by tau^2) as opposed to sampling variation. Metric is therefore a description of data in sample, not an underlying quality associated with the true effect. Can be thought of as analogous to the reliability of a scale, which also represents the percentage of true variance as portion of total variance. Range 0-100, lower values preferable. E.g., 0 = no variability in observed effects to be explained as a systematic influence (e.g., some due to some moderator) as its all just sampling variation. Does not depend on the effect size scale or the number of studies.
• H^2: Total variability / sampling variability, i.e., (true variability / sampling variability)/sampling variability. Lower values preferable, i.e., refer to less true variation to be explained as systematic (e.g., due to some moderator).

H1: IAT

results_for_meta_analysis_h1 <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", "Experiment 6",
                            "Experiment 8 (standard)", 
                            "Experiment 8 (random)"), 
             cohens_d = c(results_exp_1_h1$cohens_d$estimate[[1]], 
                          results_exp_2_h1$cohens_d$estimate[[1]], 
                          results_exp_3_h1$cohens_d$estimate[[1]], 
                          results_exp_4_h1$cohens_d$estimate[[1]],
                          results_exp_5_h1$cohens_d$estimate[[1]],
                          results_exp_6_h1$cohens_d$estimate[[1]],
                          results_exp_8nri_h1$cohens_d$estimate[[1]], 
                          results_exp_8ri_h1$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h1$cohens_d$conf.int[["lower"]],
                        results_exp_2_h1$cohens_d$conf.int[["lower"]],
                        results_exp_3_h1$cohens_d$conf.int[["lower"]],
                        results_exp_4_h1$cohens_d$conf.int[["lower"]],
                        results_exp_5_h1$cohens_d$conf.int[["lower"]],
                        results_exp_6_h1$cohens_d$conf.int[["lower"]],
                        results_exp_8nri_h1$cohens_d$conf.int[["lower"]],
                        results_exp_8ri_h1$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h1$cohens_d$conf.int[["upper"]],
                        results_exp_2_h1$cohens_d$conf.int[["upper"]],
                        results_exp_3_h1$cohens_d$conf.int[["upper"]],
                        results_exp_4_h1$cohens_d$conf.int[["upper"]],
                        results_exp_5_h1$cohens_d$conf.int[["upper"]],
                        results_exp_6_h1$cohens_d$conf.int[["upper"]],
                        results_exp_8nri_h1$cohens_d$conf.int[["upper"]],
                        results_exp_8ri_h1$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2))

results_for_meta_analysis_h1 %>%
  select(Experiment, yi, sei) %>%
  write_csv("../meta analysis data/results_for_meta_analysis_iat.csv")

meta_results_h1 <- meta_analysis_workflow(data              = results_for_meta_analysis_h1,
                                          effect_size_label = "Cohen's d",
                                          exp_estimates     = FALSE,
                                          reference_line    = 0)

• Results demonstrated a significant effect of medium size (Cohen, 1988): k = 8, Cohen’s d = 0.75, 95% CI = [0.42, 1.07], 95% CR = [-0.14, 1.63], p < .0001
• Results were found to contain a high degree of heterogeneity: Q(df = 7) = 36.81, p < .001, tau^2 = 0.18, I^2 = 83.28%, H^2 = 5.98

Outlier tests

Undue influence of one or more studies on either the meta analysed effect size or the heterogeneity among studies was then assessed.

Note on influence metrics from the metafor package documentation:

• The DFFITS value indicates how many standard deviations the predicted (average) effect for the ith study changes after excluding the ith study from the model fitting.
• The leave-one-out amount of (residual) heterogeneity is the estimated value of τ based on the dataset with the ith study removed. Note that this is always equal to 0 for fixed-effects models.
• Similarly, the leave-one-out test statistic for the test of (residual) heterogeneity is the value of the test statistic of the test for (residual) heterogeneity calculated based on the dataset with the ith study removed.

Outliers were assessed on the baiss of undue influence, on the predicted average effect size/coefficients (DFFITS), and the large decrease in heterogeneity on the basis of its removal (tau2.del, QE.del). Note that outliers were also selected based on consistency across all three DVs (see other analyses below).

influence_meta_results_h1 <- influence(meta_results_h1$fitted_model)

print(influence_meta_results_h1, digits = 2) 

## 
##                         rstudent dffits cook.d cov.r tau2.del QE.del  hat 
## Experiment 1                1.35   0.51   0.23  1.01     0.15  29.06 0.12 
## Experiment 2               -3.84  -1.25   0.51  0.36     0.03  11.24 0.13 
## Experiment 3               -0.18  -0.08   0.01  1.33     0.21  36.70 0.12 
## Experiment 4               -0.03  -0.02   0.00  1.37     0.22  36.81 0.14 
## Experiment 5               -0.39  -0.16   0.03  1.34     0.21  35.54 0.14 
## Experiment 6                0.47   0.18   0.04  1.32     0.21  33.41 0.14 
## Experiment 8 (standard)     0.20   0.06   0.00  1.30     0.21  36.58 0.11 
## Experiment 8 (random)       0.97   0.33   0.11  1.13     0.18  33.74 0.11 
##                         weight  dfbs inf 
## Experiment 1             12.10  0.51     
## Experiment 2             12.56 -1.27   * 
## Experiment 3             12.30 -0.08     
## Experiment 4             13.63 -0.02     
## Experiment 5             13.58 -0.16     
## Experiment 6             13.88  0.18     
## Experiment 8 (standard)  11.33  0.06     
## Experiment 8 (random)    10.61  0.33

Experiment 2 was assessed as being an outlier.

Sensitivity analysis

A sensitivity analysis therefore excluded Experiment 2 and re meta analysed.

results_for_meta_analysis_h1_sensitivity <- results_for_meta_analysis_h1 %>%
  filter(Experiment != "Experiment 2")

meta_results_h1_sensitivity <- 
  meta_analysis_workflow(data              = results_for_meta_analysis_h1_sensitivity,
                         effect_size_label = "Cohen's d",
                         exp_estimates     = FALSE,
                         reference_line    = 0)

• After excluding one experiment as an outlier, the meta analysed effect size was found to be of large size: k = 7, Cohen’s d = 0.86, 95% CI = [0.67, 1.06], 95% CR = [0.47, 1.26], p < .0001, and heterogeneity was found to lower, Q(df = 6) = 11.24, p < .001, tau^2 = 0.03, I^2 = 47.02%, H^2 = 1.89.

H2: Self-report ratings

results_for_meta_analysis_h2 <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", "Experiment 6",
                            "Experiment 7",
                            "Experiment 8 (standard)", 
                            "Experiment 8 (random)"),
             cohens_d = c(results_exp_1_h2$cohens_d$estimate[[1]], 
                          results_exp_2_h2$cohens_d$estimate[[1]], 
                          results_exp_3_h2$cohens_d$estimate[[1]], 
                          results_exp_4_h2$cohens_d$estimate[[1]],
                          results_exp_5_h2$cohens_d$estimate[[1]],
                          results_exp_6_h2$cohens_d$estimate[[1]],
                          results_exp_7_h2$cohens_d$estimate[[1]],
                          results_exp_8nri_h2$cohens_d$estimate[[1]],
                          results_exp_8ri_h2$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h2$cohens_d$conf.int[["lower"]],
                        results_exp_2_h2$cohens_d$conf.int[["lower"]],
                        results_exp_3_h2$cohens_d$conf.int[["lower"]],
                        results_exp_4_h2$cohens_d$conf.int[["lower"]],
                        results_exp_5_h2$cohens_d$conf.int[["lower"]],
                        results_exp_6_h2$cohens_d$conf.int[["lower"]],
                        results_exp_7_h2$cohens_d$conf.int[["lower"]],
                        results_exp_8nri_h2$cohens_d$conf.int[["lower"]],
                        results_exp_8ri_h2$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h2$cohens_d$conf.int[["upper"]],
                        results_exp_2_h2$cohens_d$conf.int[["upper"]],
                        results_exp_3_h2$cohens_d$conf.int[["upper"]],
                        results_exp_4_h2$cohens_d$conf.int[["upper"]],
                        results_exp_5_h2$cohens_d$conf.int[["upper"]],
                        results_exp_6_h2$cohens_d$conf.int[["upper"]],
                        results_exp_7_h2$cohens_d$conf.int[["upper"]],
                        results_exp_8nri_h2$cohens_d$conf.int[["upper"]],
                        results_exp_8ri_h2$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2))

results_for_meta_analysis_h2 %>%
  select(Experiment, yi, sei) %>%
  write_csv("../meta analysis data/results_for_meta_analysis_self_report.csv")

meta_results_h2 <- meta_analysis_workflow(data              = results_for_meta_analysis_h2,
                                          effect_size_label = "Cohen's d",
                                          exp_estimates     = FALSE,
                                          reference_line    = 0)

• Results demonstrated a significant effect of large size: k = 9, Cohen’s d = 0.99, 95% CI = [0.62, 1.37], 95% CR = [-0.14, 2.12], p < .0001
• Results were found to contain a high degree of heterogeneity: Q(df = 8) = 68.37, p < .001, tau^2 = 0.29, I^2 = 90.65%, H^2 = 10.7

Outlier tests

influence_meta_results_h2 <- influence(meta_results_h2$fitted_model)

print(influence_meta_results_h2, digits = 2) 

## 
##                         rstudent dffits cook.d cov.r tau2.del QE.del  hat 
## Experiment 1                1.20   0.41   0.16  1.07     0.28  62.11 0.11 
## Experiment 2               -3.60  -1.39   0.75  0.43     0.09  22.72 0.11 
## Experiment 3                0.23   0.09   0.01  1.27     0.34  68.34 0.11 
## Experiment 4                0.45   0.17   0.03  1.27     0.33  67.28 0.12 
## Experiment 5                0.28   0.11   0.01  1.29     0.34  68.20 0.12 
## Experiment 6                1.13   0.41   0.16  1.10     0.28  55.98 0.12 
## Experiment 7                0.29   0.11   0.01  1.30     0.34  67.71 0.12 
## Experiment 8 (standard)    -0.78  -0.26   0.07  1.17     0.31  63.26 0.10 
## Experiment 8 (random)      -0.66  -0.22   0.05  1.19     0.32  64.94 0.10 
##                         weight  dfbs inf 
## Experiment 1             10.65  0.41     
## Experiment 2             11.07 -1.40   * 
## Experiment 3             10.78  0.08     
## Experiment 4             11.56  0.17     
## Experiment 5             11.54  0.11     
## Experiment 6             11.67  0.41     
## Experiment 7             12.18  0.11     
## Experiment 8 (standard)  10.43 -0.26     
## Experiment 8 (random)    10.12 -0.22

Experiment 2 was assessed as being an outlier.

Sensitivity analysis

A sensitivity analysis excluded Experiment 2 and re meta analysed.

results_for_meta_analysis_h2_sensitivity <- results_for_meta_analysis_h2 %>%
  filter(Experiment != "Experiment 2")

meta_results_h2_sensitivity <- 
  meta_analysis_workflow(data              = results_for_meta_analysis_h2_sensitivity,
                         effect_size_label = "Cohen's d",
                         exp_estimates     = FALSE,
                         reference_line    = 0)

• After excluding one experiment as an outlier, the meta analysed effect size was found to be of very large size (Sawilowsky, 2009): k = 8, Cohen’s d = 1.16, 95% CI = [0.91, 1.41], 95% CR = [0.52, 1.8], p < .0001, and heterogeneity was found to very negligable, Q(df = 7) = 22.72, p < .001, tau^2 = 0.09, I^2 = 75.08%, H^2 = 4.01.

H3: Behavioural intentions

results_for_meta_analysis_h3 <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", "Experiment 4", 
                            "Experiment 5", "Experiment 6", "Experiment 7"),
             logOR = c(results_exp_1_h3$logOR[results_exp_1_h3$var == "Struan"], 
                       results_exp_2_h3$logOR[results_exp_2_h3$var == "Struan"],
                       results_exp_3_h3$logOR[results_exp_3_h3$var == "Struan"],
                       results_exp_4_h3$logOR[results_exp_4_h3$var == "Struan"],
                       results_exp_5_h3$logOR[results_exp_5_h3$var == "Struan"],
                       results_exp_6_h3$logOR[results_exp_6_h3$var == "Struan"],
                       results_exp_7_h3$logOR[results_exp_7_h3$var == "Struan"]),
             logOR_se = c(results_exp_1_h3$logOR_se[results_exp_1_h3$var == "Struan"],
                          results_exp_2_h3$logOR_se[results_exp_2_h3$var == "Struan"],
                          results_exp_3_h3$logOR_se[results_exp_3_h3$var == "Struan"],
                          results_exp_4_h3$logOR_se[results_exp_4_h3$var == "Struan"],
                          results_exp_5_h3$logOR_se[results_exp_5_h3$var == "Struan"],
                          results_exp_6_h3$logOR_se[results_exp_6_h3$var == "Struan"],
                          results_exp_7_h3$logOR_se[results_exp_7_h3$var == "Struan"])) %>%
  mutate(yi  = logOR,
         sei = logOR_se)

results_for_meta_analysis_h3 %>%
  select(Experiment, yi, sei) %>%
  write_csv("../meta analysis data/results_for_meta_analysis_behavioural_intentions.csv")

meta_results_h3 <- meta_analysis_workflow(data = results_for_meta_analysis_h3,
                                          effect_size_label = "Odds Ratio",
                                          exp_estimates     = TRUE,
                                          reference_line    = 0)

• Results demonstrated a significant effect of medium size (Chen, Cohen, & Chen, 2010): k = 7, Odds Ratio = 4.71, 95% CI = [1.7, 13.2], 95% CR = [0.31, 72.97], p = .0031
• Results were found to contain a high degree of heterogeneity: Q(df = 6) = 42.25, p = 0.003, tau^2 = 1.67, I^2 = 89.78%, H^2 = 9.79

Outlier tests

influence_meta_results_h3 <- influence(meta_results_h3$fitted_model)

print(influence_meta_results_h3, digits = 2) 

## 
##              rstudent dffits cook.d cov.r tau2.del QE.del  hat weight  dfbs inf 
## Experiment 1     0.75   0.29   0.09  1.24     1.83  39.97 0.13  12.83  0.29     
## Experiment 2    -4.78  -1.67   0.67  0.22     0.16   9.31 0.14  13.81 -1.86   * 
## Experiment 3     0.27   0.11   0.01  1.37     2.03  41.95 0.13  13.31  0.11     
## Experiment 4     0.35   0.14   0.02  1.39     2.03  41.27 0.15  14.61  0.14     
## Experiment 5     0.04   0.02   0.00  1.42     2.09  42.25 0.14  14.38  0.02     
## Experiment 6     0.90   0.38   0.15  1.22     1.74  32.68 0.15  15.10  0.38     
## Experiment 7    -0.06  -0.03   0.00  1.46     2.12  41.59 0.16  15.95 -0.03

Experiment 2 was assessed as being an outlier.

Sensitivity analysis

A sensitivity analysis excluded Experiment 2 and re meta analysed.

results_for_meta_analysis_h3_sensitivity <- results_for_meta_analysis_h3 %>%
  filter(Experiment != "Experiment 2")

meta_results_h3_sensitivity <- 
  meta_analysis_workflow(data              = results_for_meta_analysis_h3_sensitivity,
                         effect_size_label = "Odds Ratio",
                         exp_estimates     = TRUE,
                         reference_line    = 0)

• After excluding one experiment as an outlier, the meta analysed effect size was found to be of large size: k = 6, Odds Ratio = 7.24, 95% CI = [4.48, 11.7], 95% CR = [2.92, 17.99], p < .0001, and heterogeneity was found to be negligable, Q(df = 5) = 9.31, p < .001, tau^2 = 0.16, I^2 = 46.44%, H^2 = 1.87.

Robustness tests via meta analyses

H1: IAT

Refit meta analyses after excluding subsets of participants based on the exploratory awareness variables.

Note that these robustness tests also take the influence tests into account by excluding Experiment 2.

# re run tests with desired subsets of participants
## exp 1
results_exp_1_h1_us_contingency_memory <- data_exp_1 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h1_color_valence_contingency_memory <- data_exp_1 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h1_demand_compliance <- data_exp_1 %>%
  filter(demand_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h1_reactance_IAT <- data_exp_1 %>%
  filter(reactance_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 2
results_exp_2_h1_us_contingency_memory <- data_exp_2 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h1_color_valence_contingency_memory <- data_exp_2 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h1_demand_compliance <- data_exp_2 %>%
  filter(demand_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h1_reactance_IAT <- data_exp_2 %>%
  filter(reactance_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 3
results_exp_3_h1_us_contingency_memory <- data_exp_3 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h1_color_valence_contingency_memory <- data_exp_3 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h1_demand_compliance <- data_exp_3 %>%
  filter(demand_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h1_reactance_IAT <- data_exp_3 %>%
  filter(reactance_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 4
results_exp_4_h1_us_contingency_memory <- data_exp_4 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h1_color_valence_contingency_memory <- data_exp_4 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h1_demand_compliance <- data_exp_4 %>%
  filter(demand_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h1_reactance_IAT <- data_exp_4 %>%
  filter(reactance_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 5
results_exp_5_h1_us_contingency_memory <- data_exp_5 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h1_color_valence_contingency_memory <- data_exp_5 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h1_demand_compliance <- data_exp_5 %>%
  filter(demand_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h1_reactance_IAT <- data_exp_5 %>%
  filter(reactance_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 6
results_exp_6_h1_demand_compliance <- data_exp_6 %>%
  filter(demand_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_6_h1_reactance_IAT <- data_exp_6 %>%
  filter(reactance_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)


## exp 8 standard instructions
results_exp_8nri_h1_us_contingency_memory <- data_exp_8nri %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_8nri_h1_demand_compliance <- data_exp_8nri %>%
  filter(demand_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_8nri_h1_reactance_IAT <- data_exp_8nri %>%
  filter(reactance_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)


## exp 8 random instructions
results_exp_8ri_h1_us_contingency_memory <- data_exp_8ri %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_8ri_h1_demand_compliance <- data_exp_8ri %>%
  filter(demand_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_8ri_h1_reactance_IAT <- data_exp_8ri %>%
  filter(reactance_IAT == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

# US contingency_memory
results_for_meta_analysis_h1_us_contingency_memory <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5",
                            "Experiment 8 (standard)", 
                            "Experiment 8 (random)"), 
             cohens_d = c(results_exp_1_h1_us_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_2_h1_us_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_3_h1_us_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_4_h1_us_contingency_memory$cohens_d$estimate[[1]],
                          results_exp_5_h1_us_contingency_memory$cohens_d$estimate[[1]],
                          results_exp_8nri_h1_us_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_8ri_h1_us_contingency_memory$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h1_us_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_2_h1_us_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_3_h1_us_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_4_h1_us_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_5_h1_us_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_8nri_h1_us_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_8ri_h1_us_contingency_memory$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h1_us_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_2_h1_us_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_3_h1_us_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_4_h1_us_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_5_h1_us_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_8nri_h1_us_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_8ri_h1_us_contingency_memory$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2")

# color valence contingency_memory
results_for_meta_analysis_h1_color_valence_contingency_memory <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5"), 
             cohens_d = c(results_exp_1_h1_color_valence_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_2_h1_color_valence_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_3_h1_color_valence_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_4_h1_color_valence_contingency_memory$cohens_d$estimate[[1]],
                          results_exp_5_h1_color_valence_contingency_memory$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h1_color_valence_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_2_h1_color_valence_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_3_h1_color_valence_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_4_h1_color_valence_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_5_h1_color_valence_contingency_memory$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h1_color_valence_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_2_h1_color_valence_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_3_h1_color_valence_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_4_h1_color_valence_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_5_h1_color_valence_contingency_memory$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2")

# demand_compliance
results_for_meta_analysis_h1_demand_compliance <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", "Experiment 6",
                            "Experiment 8 (standard)", 
                            "Experiment 8 (random)"), 
             cohens_d = c(results_exp_1_h1_demand_compliance$cohens_d$estimate[[1]], 
                          results_exp_2_h1_demand_compliance$cohens_d$estimate[[1]], 
                          results_exp_3_h1_demand_compliance$cohens_d$estimate[[1]], 
                          results_exp_4_h1_demand_compliance$cohens_d$estimate[[1]],
                          results_exp_5_h1_demand_compliance$cohens_d$estimate[[1]],
                          results_exp_6_h1_demand_compliance$cohens_d$estimate[[1]],
                          results_exp_8nri_h1_demand_compliance$cohens_d$estimate[[1]], 
                          results_exp_8ri_h1_demand_compliance$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h1_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_2_h1_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_3_h1_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_4_h1_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_5_h1_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_6_h1_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_8nri_h1_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_8ri_h1_demand_compliance$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h1_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_2_h1_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_3_h1_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_4_h1_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_5_h1_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_6_h1_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_8nri_h1_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_8ri_h1_demand_compliance$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2")

# reactance
results_for_meta_analysis_h1_reactance_IAT <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", "Experiment 6",
                            "Experiment 8 (standard)", 
                            "Experiment 8 (random)"), 
             cohens_d = c(results_exp_1_h1_reactance_IAT$cohens_d$estimate[[1]], 
                          results_exp_2_h1_reactance_IAT$cohens_d$estimate[[1]], 
                          results_exp_3_h1_reactance_IAT$cohens_d$estimate[[1]], 
                          results_exp_4_h1_reactance_IAT$cohens_d$estimate[[1]],
                          results_exp_5_h1_reactance_IAT$cohens_d$estimate[[1]],
                          results_exp_6_h1_reactance_IAT$cohens_d$estimate[[1]],
                          results_exp_8nri_h1_reactance_IAT$cohens_d$estimate[[1]], 
                          results_exp_8ri_h1_reactance_IAT$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h1_reactance_IAT$cohens_d$conf.int[["lower"]],
                        results_exp_2_h1_reactance_IAT$cohens_d$conf.int[["lower"]],
                        results_exp_3_h1_reactance_IAT$cohens_d$conf.int[["lower"]],
                        results_exp_4_h1_reactance_IAT$cohens_d$conf.int[["lower"]],
                        results_exp_5_h1_reactance_IAT$cohens_d$conf.int[["lower"]],
                        results_exp_6_h1_reactance_IAT$cohens_d$conf.int[["lower"]],
                        results_exp_8nri_h1_reactance_IAT$cohens_d$conf.int[["lower"]],
                        results_exp_8ri_h1_reactance_IAT$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h1_reactance_IAT$cohens_d$conf.int[["upper"]],
                        results_exp_2_h1_reactance_IAT$cohens_d$conf.int[["upper"]],
                        results_exp_3_h1_reactance_IAT$cohens_d$conf.int[["upper"]],
                        results_exp_4_h1_reactance_IAT$cohens_d$conf.int[["upper"]],
                        results_exp_5_h1_reactance_IAT$cohens_d$conf.int[["upper"]],
                        results_exp_6_h1_reactance_IAT$cohens_d$conf.int[["upper"]],
                        results_exp_8nri_h1_reactance_IAT$cohens_d$conf.int[["upper"]],
                        results_exp_8ri_h1_reactance_IAT$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2")

# fit metas
meta_results_h1_us_contingency_memory <- 
  meta_analysis_workflow(data              = results_for_meta_analysis_h1_us_contingency_memory,
                         effect_size_label = "Cohen's d",
                         exp_estimates     = FALSE,
                         reference_line    = 0,
                         plot = FALSE)

meta_results_h1_color_valence_contingency_memory <- 
  meta_analysis_workflow(data              = results_for_meta_analysis_h1_color_valence_contingency_memory,
                         effect_size_label = "Cohen's d",
                         exp_estimates     = FALSE,
                         reference_line    = 0,
                         plot = FALSE)

meta_results_h1_demand_compliance <- 
  meta_analysis_workflow(data              = results_for_meta_analysis_h1_demand_compliance,
                         effect_size_label = "Cohen's d",
                         exp_estimates     = FALSE,
                         reference_line    = 0,
                         plot = FALSE)

meta_results_h1_reactance_IAT <- 
  meta_analysis_workflow(data              = results_for_meta_analysis_h1_reactance_IAT,
                         effect_size_label = "Cohen's d",
                         exp_estimates     = FALSE,
                         reference_line    = 0,
                         plot = FALSE)

# robustness strings
robustness_us_contingency_memory_h1 <-
  ifelse(robust_test(meta_results_h1_us_contingency_memory$meta_analysis_results, 
                     meta_results_h1$meta_analysis_results),
         "were", "were not")

robustness_color_valence_contingency_memory_h1 <-
  ifelse(robust_test(meta_results_h1_color_valence_contingency_memory$meta_analysis_results, 
                     meta_results_h1$meta_analysis_results),
         "were", "were not")

robustness_demand_compliance_h1  <-
  ifelse(robust_test(meta_results_h1_demand_compliance$meta_analysis_results, 
                     meta_results_h1$meta_analysis_results),
         "were", "were not")

robustness_reactance_IAT_excluded_h1    <-
  ifelse(robust_test(meta_results_h1_reactance_IAT$meta_analysis_results, 
                     meta_results_h1$meta_analysis_results),
         "were", "were not")

• Across studies, results were robust to including only participants who were US contingency aware, p in subsample < .0001.
• Across studies, results were robust to including only participants who were color-valence contingency aware, p in subsample = .0016.
• Across studies, results were robust to including only participants who were not demand compliant, p in subsample < .0001.
• Across studies, results were robust to including only participants who were not reactant, p in subsample < .0001.

H2: Self-report ratings

# re run tests with desired subsets of participants
## exp 1
results_exp_1_h2_us_contingency_memory <- data_exp_1 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h2_color_valence_contingency_memory <- data_exp_1 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h2_demand_compliance <- data_exp_1 %>%
  filter(demand_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h2_reactance_self_reports <- data_exp_1 %>%
  filter(reactance_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 2
results_exp_2_h2_us_contingency_memory <- data_exp_2 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h2_color_valence_contingency_memory <- data_exp_2 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h2_demand_compliance <- data_exp_2 %>%
  filter(demand_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h2_reactance_self_reports <- data_exp_2 %>%
  filter(reactance_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 3
results_exp_3_h2_us_contingency_memory <- data_exp_3 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h2_color_valence_contingency_memory <- data_exp_3 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h2_demand_compliance <- data_exp_3 %>%
  filter(demand_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h2_reactance_self_reports <- data_exp_3 %>%
  filter(reactance_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 4
results_exp_4_h2_us_contingency_memory <- data_exp_4 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h2_color_valence_contingency_memory <- data_exp_4 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h2_demand_compliance <- data_exp_4 %>%
  filter(demand_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h2_reactance_self_reports <- data_exp_4 %>%
  filter(reactance_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 5
results_exp_5_h2_us_contingency_memory <- data_exp_5 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h2_color_valence_contingency_memory <- data_exp_5 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h2_demand_compliance <- data_exp_5 %>%
  filter(demand_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h2_reactance_self_reports <- data_exp_5 %>%
  filter(reactance_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 6
results_exp_6_h2_demand_compliance <- data_exp_6 %>%
  filter(demand_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_6_h2_reactance_self_reports <- data_exp_6 %>%
  filter(reactance_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)


## exp 7
results_exp_7_h2_us_contingency_memory <- data_exp_7 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_7_h2_color_valence_contingency_memory <- data_exp_7 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_7_h2_demand_compliance <- data_exp_7 %>%
  filter(demand_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_7_h2_reactance_self_reports <- data_exp_7 %>%
  filter(reactance_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)


## exp 8 standard instructions
results_exp_8nri_h2_us_contingency_memory <- data_exp_8nri %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_8nri_h2_demand_compliance <- data_exp_8nri %>%
  filter(demand_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_8nri_h2_reactance_self_reports <- data_exp_8nri %>%
  filter(reactance_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)


## exp 8 random instructions
results_exp_8ri_h2_us_contingency_memory <- data_exp_8ri %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_8ri_h2_demand_compliance <- data_exp_8ri %>%
  filter(demand_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_8ri_h2_reactance_self_reports <- data_exp_8ri %>%
  filter(reactance_self_reports == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)


# US contingency_memory
results_for_meta_analysis_h2_us_contingency_memory <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", 
                            "Experiment 7",
                            "Experiment 8 (standard)", 
                            "Experiment 8 (random)"), 
             cohens_d = c(results_exp_1_h2_us_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_2_h2_us_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_3_h2_us_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_4_h2_us_contingency_memory$cohens_d$estimate[[1]],
                          results_exp_5_h2_us_contingency_memory$cohens_d$estimate[[1]],
                          results_exp_7_h2_us_contingency_memory$cohens_d$estimate[[1]],
                          results_exp_8nri_h2_us_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_8ri_h2_us_contingency_memory$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h2_us_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_2_h2_us_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_3_h2_us_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_4_h2_us_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_5_h2_us_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_7_h2_us_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_8nri_h2_us_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_8ri_h2_us_contingency_memory$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h2_us_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_2_h2_us_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_3_h2_us_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_4_h2_us_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_5_h2_us_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_7_h2_us_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_8nri_h2_us_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_8ri_h2_us_contingency_memory$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2")

# color valence contingency_memory
results_for_meta_analysis_h2_color_valence_contingency_memory <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", 
                            "Experiment 7"), 
             cohens_d = c(results_exp_1_h2_color_valence_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_2_h2_color_valence_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_3_h2_color_valence_contingency_memory$cohens_d$estimate[[1]], 
                          results_exp_4_h2_color_valence_contingency_memory$cohens_d$estimate[[1]],
                          results_exp_5_h2_color_valence_contingency_memory$cohens_d$estimate[[1]],
                          results_exp_7_h2_color_valence_contingency_memory$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h2_color_valence_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_2_h2_color_valence_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_3_h2_color_valence_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_4_h2_color_valence_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_5_h2_color_valence_contingency_memory$cohens_d$conf.int[["lower"]],
                        results_exp_7_h2_color_valence_contingency_memory$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h2_color_valence_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_2_h2_color_valence_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_3_h2_color_valence_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_4_h2_color_valence_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_5_h2_color_valence_contingency_memory$cohens_d$conf.int[["upper"]],
                        results_exp_7_h2_color_valence_contingency_memory$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2")

# demand_compliance
results_for_meta_analysis_h2_demand_compliance <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", "Experiment 6",
                            "Experiment 7",
                            "Experiment 8 (standard)", 
                            "Experiment 8 (random)"), 
             cohens_d = c(results_exp_1_h2_demand_compliance$cohens_d$estimate[[1]], 
                          results_exp_2_h2_demand_compliance$cohens_d$estimate[[1]], 
                          results_exp_3_h2_demand_compliance$cohens_d$estimate[[1]], 
                          results_exp_4_h2_demand_compliance$cohens_d$estimate[[1]],
                          results_exp_5_h2_demand_compliance$cohens_d$estimate[[1]],
                          results_exp_6_h2_demand_compliance$cohens_d$estimate[[1]],
                          results_exp_7_h2_demand_compliance$cohens_d$estimate[[1]],
                          results_exp_8nri_h2_demand_compliance$cohens_d$estimate[[1]], 
                          results_exp_8ri_h2_demand_compliance$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h2_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_2_h2_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_3_h2_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_4_h2_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_5_h2_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_6_h2_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_7_h2_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_8nri_h2_demand_compliance$cohens_d$conf.int[["lower"]],
                        results_exp_8ri_h2_demand_compliance$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h2_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_2_h2_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_3_h2_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_4_h2_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_5_h2_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_6_h2_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_7_h2_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_8nri_h2_demand_compliance$cohens_d$conf.int[["upper"]],
                        results_exp_8ri_h2_demand_compliance$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2")

# reactance_self_reports
results_for_meta_analysis_h2_reactance_self_reports <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", "Experiment 6",
                            "Experiment 7",
                            "Experiment 8 (standard)", 
                            "Experiment 8 (random)"), 
             cohens_d = c(results_exp_1_h2_reactance_self_reports$cohens_d$estimate[[1]], 
                          results_exp_2_h2_reactance_self_reports$cohens_d$estimate[[1]], 
                          results_exp_3_h2_reactance_self_reports$cohens_d$estimate[[1]], 
                          results_exp_4_h2_reactance_self_reports$cohens_d$estimate[[1]],
                          results_exp_5_h2_reactance_self_reports$cohens_d$estimate[[1]],
                          results_exp_6_h2_reactance_self_reports$cohens_d$estimate[[1]],
                          results_exp_7_h2_reactance_self_reports$cohens_d$estimate[[1]],
                          results_exp_8nri_h2_reactance_self_reports$cohens_d$estimate[[1]], 
                          results_exp_8ri_h2_reactance_self_reports$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h2_reactance_self_reports$cohens_d$conf.int[["lower"]],
                        results_exp_2_h2_reactance_self_reports$cohens_d$conf.int[["lower"]],
                        results_exp_3_h2_reactance_self_reports$cohens_d$conf.int[["lower"]],
                        results_exp_4_h2_reactance_self_reports$cohens_d$conf.int[["lower"]],
                        results_exp_5_h2_reactance_self_reports$cohens_d$conf.int[["lower"]],
                        results_exp_6_h2_reactance_self_reports$cohens_d$conf.int[["lower"]],
                        results_exp_7_h2_reactance_self_reports$cohens_d$conf.int[["lower"]],
                        results_exp_8nri_h2_reactance_self_reports$cohens_d$conf.int[["lower"]],
                        results_exp_8ri_h2_reactance_self_reports$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h2_reactance_self_reports$cohens_d$conf.int[["upper"]],
                        results_exp_2_h2_reactance_self_reports$cohens_d$conf.int[["upper"]],
                        results_exp_3_h2_reactance_self_reports$cohens_d$conf.int[["upper"]],
                        results_exp_4_h2_reactance_self_reports$cohens_d$conf.int[["upper"]],
                        results_exp_5_h2_reactance_self_reports$cohens_d$conf.int[["upper"]],
                        results_exp_6_h2_reactance_self_reports$cohens_d$conf.int[["upper"]],
                        results_exp_7_h2_reactance_self_reports$cohens_d$conf.int[["upper"]],
                        results_exp_8nri_h2_reactance_self_reports$cohens_d$conf.int[["upper"]],
                        results_exp_8ri_h2_reactance_self_reports$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2")

# fit metas
meta_results_h2_us_contingency_memory <- 
  meta_analysis_workflow(data              = results_for_meta_analysis_h2_us_contingency_memory,
                         effect_size_label = "Cohen's d",
                         exp_estimates     = FALSE,
                         reference_line    = 0,
                         plot = FALSE)

meta_results_h2_color_valence_contingency_memory <- 
  meta_analysis_workflow(data              = results_for_meta_analysis_h2_color_valence_contingency_memory,
                         effect_size_label = "Cohen's d",
                         exp_estimates     = FALSE,
                         reference_line    = 0,
                         plot = FALSE)

meta_results_h2_demand_compliance <- 
  meta_analysis_workflow(data              = results_for_meta_analysis_h2_demand_compliance,
                         effect_size_label = "Cohen's d",
                         exp_estimates     = FALSE,
                         reference_line    = 0,
                         plot = FALSE)

meta_results_h2_reactance_self_reports <- 
  meta_analysis_workflow(data              = results_for_meta_analysis_h2_reactance_self_reports,
                         effect_size_label = "Cohen's d",
                         exp_estimates     = FALSE,
                         reference_line    = 0,
                         plot = FALSE)

# robustness strings
robustness_us_contingency_memory_h2 <-
  ifelse(robust_test(meta_results_h2_us_contingency_memory$meta_analysis_results, 
                     meta_results_h2$meta_analysis_results),
         "were", "were not")

robustness_color_valence_contingency_memory_h2 <-
  ifelse(robust_test(meta_results_h2_color_valence_contingency_memory$meta_analysis_results, 
                     meta_results_h2$meta_analysis_results),
         "were", "were not")

robustness_demand_compliance_h2  <-
  ifelse(robust_test(meta_results_h2_demand_compliance$meta_analysis_results, 
                     meta_results_h2$meta_analysis_results),
         "were", "were not")

robustness_reactance_self_reports_h2    <-
  ifelse(robust_test(meta_results_h2_reactance_self_reports$meta_analysis_results, 
                     meta_results_h2$meta_analysis_results),
         "were", "were not")

• Across studies, results were robust to including only participants who were US contingency aware, p in subsample < .0001.
• Across studies, results were robust to including only participants who were color-valence contingency aware, p in subsample < .0001.
• Across studies, results were robust to including only participants who were not demand compliant, p in subsample < .0001.
• Across studies, results were robust to including only participants who were not reactant, p in subsample < .0001.

H3: Behavioural intentions

# re run tests with desired subsets of participants
# exp 1
results_exp_1_h3_us_contingency_memory <- data_exp_1 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_1_h3_color_valence_contingency_memory <- data_exp_1 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

# exp 2
results_exp_2_h3_us_contingency_memory <- data_exp_2 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_2_h3_color_valence_contingency_memory <- data_exp_2 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

# exp 3
results_exp_3_h3_us_contingency_memory <- data_exp_3 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_3_h3_color_valence_contingency_memory <- data_exp_3 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

# exp 4
results_exp_4_h3_us_contingency_memory <- data_exp_4 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_4_h3_color_valence_contingency_memory <- data_exp_4 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

# exp 5
results_exp_5_h3_us_contingency_memory <- data_exp_5 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_5_h3_color_valence_contingency_memory <- data_exp_5 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

# exp 7
results_exp_7_h3_us_contingency_memory <- data_exp_7 %>%
  filter(US_contingency_memory == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_7_h3_color_valence_contingency_memory <- data_exp_7 %>%
  filter(color_valence_contingency_memory == TRUE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

# us contingency memory
results_for_meta_analysis_h3_us_contingency_memory <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", "Experiment 4", 
                            "Experiment 5", "Experiment 7"),
             logOR = c(results_exp_1_h3_us_contingency_memory$logOR[results_exp_1_h3_us_contingency_memory$var == "Struan"], 
                       results_exp_2_h3_us_contingency_memory$logOR[results_exp_2_h3_us_contingency_memory$var == "Struan"],
                       results_exp_3_h3_us_contingency_memory$logOR[results_exp_3_h3_us_contingency_memory$var == "Struan"],
                       results_exp_4_h3_us_contingency_memory$logOR[results_exp_4_h3_us_contingency_memory$var == "Struan"],
                       results_exp_5_h3_us_contingency_memory$logOR[results_exp_5_h3_us_contingency_memory$var == "Struan"],
                       results_exp_7_h3_us_contingency_memory$logOR[results_exp_7_h3_us_contingency_memory$var == "Struan"]),
             logOR_se = c(results_exp_1_h3_us_contingency_memory$logOR_se[results_exp_1_h3_us_contingency_memory$var == "Struan"],
                          results_exp_2_h3_us_contingency_memory$logOR_se[results_exp_2_h3_us_contingency_memory$var == "Struan"],
                          results_exp_3_h3_us_contingency_memory$logOR_se[results_exp_3_h3_us_contingency_memory$var == "Struan"],
                          results_exp_4_h3_us_contingency_memory$logOR_se[results_exp_4_h3_us_contingency_memory$var == "Struan"],
                          results_exp_5_h3_us_contingency_memory$logOR_se[results_exp_5_h3_us_contingency_memory$var == "Struan"],
                          results_exp_7_h3_us_contingency_memory$logOR_se[results_exp_7_h3_us_contingency_memory$var == "Struan"])) %>%
  mutate(yi  = logOR,
         sei = logOR_se) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2")

# color valence contingency memory
results_for_meta_analysis_h3_color_valence_contingency_memory <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", "Experiment 4", 
                            "Experiment 5", "Experiment 7"),
             logOR = c(results_exp_1_h3_color_valence_contingency_memory$logOR[results_exp_1_h3_color_valence_contingency_memory$var == "Struan"], 
                       results_exp_2_h3_color_valence_contingency_memory$logOR[results_exp_2_h3_color_valence_contingency_memory$var == "Struan"],
                       results_exp_3_h3_color_valence_contingency_memory$logOR[results_exp_3_h3_color_valence_contingency_memory$var == "Struan"],
                       results_exp_4_h3_color_valence_contingency_memory$logOR[results_exp_4_h3_color_valence_contingency_memory$var == "Struan"],
                       results_exp_5_h3_color_valence_contingency_memory$logOR[results_exp_5_h3_color_valence_contingency_memory$var == "Struan"],
                       results_exp_7_h3_color_valence_contingency_memory$logOR[results_exp_7_h3_color_valence_contingency_memory$var == "Struan"]),
             logOR_se = c(results_exp_1_h3_color_valence_contingency_memory$logOR_se[results_exp_1_h3_color_valence_contingency_memory$var == "Struan"],
                          results_exp_2_h3_color_valence_contingency_memory$logOR_se[results_exp_2_h3_color_valence_contingency_memory$var == "Struan"],
                          results_exp_3_h3_color_valence_contingency_memory$logOR_se[results_exp_3_h3_color_valence_contingency_memory$var == "Struan"],
                          results_exp_4_h3_color_valence_contingency_memory$logOR_se[results_exp_4_h3_color_valence_contingency_memory$var == "Struan"],
                          results_exp_5_h3_color_valence_contingency_memory$logOR_se[results_exp_5_h3_color_valence_contingency_memory$var == "Struan"],
                          results_exp_7_h3_color_valence_contingency_memory$logOR_se[results_exp_7_h3_color_valence_contingency_memory$var == "Struan"])) %>%
  mutate(yi  = logOR,
         sei = logOR_se) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2")

# meta fits
meta_results_h3_us_contingency_memory <- 
  meta_analysis_workflow(data = results_for_meta_analysis_h3_us_contingency_memory,
                         effect_size_label = "Odds Ratio",
                         exp_estimates     = TRUE,
                         reference_line    = 0,
                         plot = FALSE)

meta_results_h3_color_valence_contingency_memory <- 
  meta_analysis_workflow(data = results_for_meta_analysis_h3_color_valence_contingency_memory,
                         effect_size_label = "Odds Ratio",
                         exp_estimates     = TRUE,
                         reference_line    = 0,
                         plot = FALSE)

# robustness strings
robustness_us_contingency_memory_h3 <-
  ifelse(robust_test(meta_results_h3_us_contingency_memory$meta_analysis_results, 
                     meta_results_h1$meta_analysis_results),
         "were", "were not")

robustness_color_valence_contingency_memory_h3 <-
  ifelse(robust_test(meta_results_h3_color_valence_contingency_memory$meta_analysis_results, 
                     meta_results_h1$meta_analysis_results),
         "were", "were not")

• Across studies, results were robust to including only participants who were US contingency aware, p in subsample < .0001.
• Across studies, results were robust to including only participants who were color-valence contingency aware, p in subsample < .0001.

Moderation analyses

Our robustness analyses exclude participants who meet certain criteria. A reviewer requested that we also calculate modration by these variables.

H1: IAT

Note that these tests also take the influence tests into account by excluding Experiment 2.

# re run tests with desired subsets of participants
## exp 1
results_exp_1_h1_us_contingency_memory_excluded <- data_exp_1 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h1_color_valence_contingency_memory_excluded <- data_exp_1 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h1_demand_compliance_excluded <- data_exp_1 %>%
  filter(demand_IAT == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h1_reactance_IAT_excluded <- data_exp_1 %>%
  filter(reactance_IAT == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 2
results_exp_2_h1_us_contingency_memory_excluded <- data_exp_2 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h1_color_valence_contingency_memory_excluded <- data_exp_2 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h1_demand_compliance_excluded <- data_exp_2 %>%
  filter(demand_IAT == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h1_reactance_IAT_excluded <- data_exp_2 %>%
  filter(reactance_IAT == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 3
results_exp_3_h1_us_contingency_memory_excluded <- data_exp_3 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h1_color_valence_contingency_memory_excluded <- data_exp_3 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h1_demand_compliance_excluded <- data_exp_3 %>%
  filter(demand_IAT == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h1_reactance_IAT_excluded <- data_exp_3 %>%
  filter(reactance_IAT == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 4
results_exp_4_h1_us_contingency_memory_excluded <- data_exp_4 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h1_color_valence_contingency_memory_excluded <- data_exp_4 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h1_demand_compliance_excluded <- data_exp_4 %>%
  filter(demand_IAT == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h1_reactance_IAT_excluded <- data_exp_4 %>%
  filter(reactance_IAT == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 5
results_exp_5_h1_us_contingency_memory_excluded <- data_exp_5 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h1_color_valence_contingency_memory_excluded <- data_exp_5 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h1_demand_compliance_excluded <- data_exp_5 %>%
  filter(demand_IAT == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h1_reactance_IAT_excluded <- data_exp_5 %>%
  filter(reactance_IAT == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

# ## exp 6
# results_exp_6_h1_demand_compliance_excluded <- data_exp_6 %>%
#   filter(demand_IAT == TRUE) %>%
#   mutate(DV = IAT_Score,
#          IV = source_valence_condition) %>%
#   t_test_analyses(.)
#>not enough 'x' observations

results_exp_6_h1_reactance_IAT_excluded <- data_exp_6 %>%
  filter(reactance_IAT == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)


## exp 8 standard instructions
# results_exp_8nri_h1_us_contingency_memory_excluded <- data_exp_8nri %>%
#   filter(US_contingency_memory == FALSE) %>%
#   mutate(DV = IAT_Score,
#          IV = source_valence_condition) %>%
#   t_test_analyses(.)
#>not enough 'x' observations

# results_exp_8nri_h1_demand_compliance_excluded <- data_exp_8nri %>%
#   filter(demand_IAT == TRUE) %>%
#   mutate(DV = IAT_Score,
#          IV = source_valence_condition) %>%
#   t_test_analyses(.)
#>not enough 'x' observations

results_exp_8nri_h1_reactance_IAT_excluded <- data_exp_8nri %>%
  filter(reactance_IAT == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)


## exp 8 random instructions
results_exp_8ri_h1_us_contingency_memory_excluded <- data_exp_8ri %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

# results_exp_8ri_h1_demand_compliance_excluded <- data_exp_8ri %>%
#   filter(demand_IAT == TRUE) %>%
#   distinct(source_valence_condition)
#   mutate(DV = IAT_Score,
#          IV = source_valence_condition) %>%
#   t_test_analyses(.)
# not enough 'x' observations

results_exp_8ri_h1_reactance_IAT_excluded <- data_exp_8ri %>%
  filter(reactance_IAT == TRUE) %>%
  mutate(DV = IAT_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

# US contingency_memory
results_for_meta_analysis_h1_us_contingency_memory_excluded <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5",
                            #"Experiment 8 (standard)", 
                            "Experiment 8 (random)"),
             cohens_d = c(results_exp_1_h1_us_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_2_h1_us_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_3_h1_us_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_4_h1_us_contingency_memory_excluded$cohens_d$estimate[[1]],
                          results_exp_5_h1_us_contingency_memory_excluded$cohens_d$estimate[[1]],
                          #results_exp_8nri_h1_us_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_8ri_h1_us_contingency_memory_excluded$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h1_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_2_h1_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_3_h1_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_4_h1_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_5_h1_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        #results_exp_8nri_h1_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_8ri_h1_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h1_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_2_h1_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_3_h1_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_4_h1_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_5_h1_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        #results_exp_8nri_h1_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_8ri_h1_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2") %>%
  mutate(exclude = TRUE)

# color valence contingency_memory
results_for_meta_analysis_h1_color_valence_contingency_memory_excluded <- 
  data.frame(Experiment = c("Experiment 1", 
                            "Experiment 2", 
                            "Experiment 3", 
                            "Experiment 4", 
                            "Experiment 5"), 
             cohens_d = c(results_exp_1_h1_color_valence_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_2_h1_color_valence_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_3_h1_color_valence_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_4_h1_color_valence_contingency_memory_excluded$cohens_d$estimate[[1]],
                          results_exp_5_h1_color_valence_contingency_memory_excluded$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h1_color_valence_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_2_h1_color_valence_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_3_h1_color_valence_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_4_h1_color_valence_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_5_h1_color_valence_contingency_memory_excluded$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h1_color_valence_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_2_h1_color_valence_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_3_h1_color_valence_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_4_h1_color_valence_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_5_h1_color_valence_contingency_memory_excluded$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2") %>%
  mutate(exclude = TRUE)

# demand_compliance
results_for_meta_analysis_h1_demand_compliance_excluded <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5"), 
             #"Experiment 6",
             #"Experiment 8 (standard)"), 
             #"Experiment 8 (random)"), 
             cohens_d = c(results_exp_1_h1_demand_compliance_excluded$cohens_d$estimate[[1]], 
                          results_exp_2_h1_demand_compliance_excluded$cohens_d$estimate[[1]], 
                          results_exp_3_h1_demand_compliance_excluded$cohens_d$estimate[[1]], 
                          results_exp_4_h1_demand_compliance_excluded$cohens_d$estimate[[1]],
                          results_exp_5_h1_demand_compliance_excluded$cohens_d$estimate[[1]]),
             #results_exp_6_h1_demand_compliance_excluded_excluded$cohens_d$estimate[[1]],
             #results_exp_8nri_h1_demand_compliance_excluded$cohens_d$estimate[[1]]), 
             #results_exp_8ri_h1_demand_compliance_excluded$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h1_demand_compliance_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_2_h1_demand_compliance_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_3_h1_demand_compliance_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_4_h1_demand_compliance_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_5_h1_demand_compliance_excluded$cohens_d$conf.int[["lower"]]),
             #results_exp_6_h1_demand_compliance_excluded$cohens_d$conf.int[["lower"]],
             #results_exp_8nri_h1_demand_compliance_excluded$cohens_d$conf.int[["lower"]]),
             #results_exp_8ri_h1_demand_compliance_excluded$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h1_demand_compliance_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_2_h1_demand_compliance_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_3_h1_demand_compliance_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_4_h1_demand_compliance_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_5_h1_demand_compliance_excluded$cohens_d$conf.int[["upper"]])) %>%
  #results_exp_6_h1_demand_compliance_excluded$cohens_d_excluded$conf.int[["upper"]],
  #results_exp_8nri_h1_demand_compliance_excluded$cohens_d$conf.int[["upper"]])) %>%
  #results_exp_8ri_h1_demand_compliance_excluded$cohens_d_excluded$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2") %>%
  mutate(exclude = TRUE)

# reactance
results_for_meta_analysis_h1_reactance_IAT_excluded <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", "Experiment 6",
                            "Experiment 8 (standard)", 
                            "Experiment 8 (random)"), 
             cohens_d = c(results_exp_1_h1_reactance_IAT_excluded$cohens_d$estimate[[1]], 
                          results_exp_2_h1_reactance_IAT_excluded$cohens_d$estimate[[1]], 
                          results_exp_3_h1_reactance_IAT_excluded$cohens_d$estimate[[1]], 
                          results_exp_4_h1_reactance_IAT_excluded$cohens_d$estimate[[1]],
                          results_exp_5_h1_reactance_IAT_excluded$cohens_d$estimate[[1]],
                          results_exp_6_h1_reactance_IAT_excluded$cohens_d$estimate[[1]],
                          results_exp_8nri_h1_reactance_IAT_excluded$cohens_d$estimate[[1]], 
                          results_exp_8ri_h1_reactance_IAT_excluded$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h1_reactance_IAT_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_2_h1_reactance_IAT_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_3_h1_reactance_IAT_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_4_h1_reactance_IAT_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_5_h1_reactance_IAT_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_6_h1_reactance_IAT_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_8nri_h1_reactance_IAT_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_8ri_h1_reactance_IAT_excluded$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h1_reactance_IAT_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_2_h1_reactance_IAT_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_3_h1_reactance_IAT_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_4_h1_reactance_IAT_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_5_h1_reactance_IAT_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_6_h1_reactance_IAT_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_8nri_h1_reactance_IAT_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_8ri_h1_reactance_IAT_excluded$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2") %>%
  mutate(exclude = TRUE)

# fit metas
mod_meta_results_h1_us_contingency_memory <- 
  multilevel_moderator_meta_analysis_workflow(data = bind_rows(semi_join(mutate(results_for_meta_analysis_h1_us_contingency_memory, exclude = FALSE),
                                                              results_for_meta_analysis_h1_us_contingency_memory_excluded,
                                                              by = "Experiment"), 
                                                    results_for_meta_analysis_h1_us_contingency_memory_excluded),
                                   effect_size_label = "Cohen's d")

mod_meta_results_h1_color_valence_contingency_memory <- 
  multilevel_moderator_meta_analysis_workflow(data = bind_rows(semi_join(mutate(results_for_meta_analysis_h1_color_valence_contingency_memory, exclude = FALSE),
                                                              results_for_meta_analysis_h1_color_valence_contingency_memory_excluded,
                                                              by = "Experiment"),
                                                    results_for_meta_analysis_h1_color_valence_contingency_memory_excluded),
                                   effect_size_label = "Cohen's d")

mod_meta_results_h1_demand_compliance <- 
  multilevel_moderator_meta_analysis_workflow(data = bind_rows(semi_join(mutate(results_for_meta_analysis_h1_demand_compliance, exclude = FALSE),
                                                              results_for_meta_analysis_h1_demand_compliance_excluded,
                                                              by = "Experiment"),
                                                    results_for_meta_analysis_h1_demand_compliance_excluded),
                                   effect_size_label = "Cohen's d")

mod_meta_results_h1_reactance <- 
  multilevel_moderator_meta_analysis_workflow(data = bind_rows(semi_join(mutate(results_for_meta_analysis_h1_reactance_IAT, exclude = FALSE),
                                                              results_for_meta_analysis_h1_reactance_IAT_excluded,
                                                              by = "Experiment"),
                                                    results_for_meta_analysis_h1_reactance_IAT_excluded),
                                   effect_size_label = "Cohen's d")

# forest plots
forest(mod_meta_results_h1_us_contingency_memory$fitted_model)

forest(mod_meta_results_h1_color_valence_contingency_memory$fitted_model)

forest(mod_meta_results_h1_demand_compliance$fitted_model)

forest(mod_meta_results_h1_reactance$fitted_model)

Moderation by:

• US contingency memory (TRUE were excluded): QM(df = 1) = 0.01, p = 0.933, difference in excluded subset Cohen’s d = 0.02, 95% CI [-0.39, 0.43]
• color-valence contingency memory (TRUE were excluded): QM(df = 1) = 1.59, p = 0.207, difference in excluded subset Cohen’s d = -0.79, 95% CI [-2, 0.43]
• demand compliance (FALSE were excluded): QM(df = 1) = 0.01, p = 0.913, difference in excluded subset Cohen’s d = -0.03, 95% CI [-0.63, 0.56]
• reactance (FALSE were excluded): QM(df = 1) = 0.01, p = 0.911, difference in excluded subset Cohen’s d = -0.03, 95% CI [-0.54, 0.48]

H2: Self-report ratings

# re run tests with desired subsets of participants
## exp 1
results_exp_1_h2_us_contingency_memory_excluded <- data_exp_1 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h2_color_valence_contingency_memory_excluded <- data_exp_1 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h2_demand_compliance_excluded <- data_exp_1 %>%
  filter(demand_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_1_h2_reactance_excluded <- data_exp_1 %>%
  filter(reactance_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 2
results_exp_2_h2_us_contingency_memory_excluded <- data_exp_2 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h2_color_valence_contingency_memory_excluded <- data_exp_2 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h2_demand_compliance_excluded <- data_exp_2 %>%
  filter(demand_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_2_h2_reactance_self_reports_excluded <- data_exp_2 %>%
  filter(reactance_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 3
results_exp_3_h2_us_contingency_memory_excluded <- data_exp_3 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h2_color_valence_contingency_memory_excluded <- data_exp_3 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h2_demand_compliance_excluded <- data_exp_3 %>%
  filter(demand_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_3_h2_reactance_self_reports_excluded <- data_exp_3 %>%
  filter(reactance_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 4
results_exp_4_h2_us_contingency_memory_excluded <- data_exp_4 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h2_color_valence_contingency_memory_excluded <- data_exp_4 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h2_demand_compliance_excluded <- data_exp_4 %>%
  filter(demand_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_4_h2_reactance_self_reports_excluded <- data_exp_4 %>%
  filter(reactance_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 5
results_exp_5_h2_us_contingency_memory_excluded <- data_exp_5 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h2_color_valence_contingency_memory_excluded <- data_exp_5 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h2_demand_compliance_excluded <- data_exp_5 %>%
  filter(demand_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_5_h2_reactance_self_reports_excluded <- data_exp_5 %>%
  filter(reactance_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

## exp 6
results_exp_6_h2_demand_compliance_excluded <- data_exp_6 %>%
  filter(demand_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_6_h2_reactance_self_reports_excluded <- data_exp_6 %>%
  filter(reactance_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)


## exp 7
results_exp_7_h2_us_contingency_memory_excluded <- data_exp_7 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_7_h2_color_valence_contingency_memory_excluded <- data_exp_7 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_7_h2_demand_compliance_excluded <- data_exp_7 %>%
  filter(demand_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_7_h2_reactance_self_reports_excluded <- data_exp_7 %>%
  filter(reactance_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)


## exp 8 standard instructions
# results_exp_8nri_h2_us_contingency_memory_excluded <- data_exp_8nri %>%
#   filter(US_contingency_memory == FALSE) %>%
#   mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
#          IV = source_valence_condition) %>%
#   t_test_analyses(.)
# > NOT ENOUGH OBSERVATIONS

results_exp_8nri_h2_demand_compliance_excluded <- data_exp_8nri %>%
  filter(demand_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_8nri_h2_reactance_self_reports_excluded <- data_exp_8nri %>%
  filter(reactance_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)


## exp 8 random instructions
results_exp_8ri_h2_us_contingency_memory_excluded <- data_exp_8ri %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_8ri_h2_demand_compliance_excluded <- data_exp_8ri %>%
  filter(demand_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)

results_exp_8ri_h2_reactance_self_reports_excluded <- data_exp_8ri %>%
  filter(reactance_self_reports == TRUE) %>%
  mutate(DV = EC_Effect_CS1_CS2_Difference_Score,
         IV = source_valence_condition) %>%
  t_test_analyses(.)


# US contingency_memory
results_for_meta_analysis_h2_us_contingency_memory_excluded <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", 
                            "Experiment 7",
                            #"Experiment 8 (standard)", 
                            "Experiment 8 (random)"), 
             cohens_d = c(results_exp_1_h2_us_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_2_h2_us_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_3_h2_us_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_4_h2_us_contingency_memory_excluded$cohens_d$estimate[[1]],
                          results_exp_5_h2_us_contingency_memory_excluded$cohens_d$estimate[[1]],
                          results_exp_7_h2_us_contingency_memory$cohens_d$estimate[[1]],
                          #results_exp_8nri_h2_us_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_8ri_h2_us_contingency_memory_excluded$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h2_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_2_h2_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_3_h2_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_4_h2_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_5_h2_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_7_h2_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        #results_exp_8nri_h2_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_8ri_h2_us_contingency_memory_excluded$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h2_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_2_h2_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_3_h2_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_4_h2_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_5_h2_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_7_h2_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        #results_exp_8nri_h2_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_8ri_h2_us_contingency_memory_excluded$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2") %>%
  mutate(exclude = TRUE)

# color valence contingency_memory
results_for_meta_analysis_h2_color_valence_contingency_memory_excluded <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", 
                            "Experiment 7"), 
             cohens_d = c(results_exp_1_h2_color_valence_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_2_h2_color_valence_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_3_h2_color_valence_contingency_memory_excluded$cohens_d$estimate[[1]], 
                          results_exp_4_h2_color_valence_contingency_memory_excluded$cohens_d$estimate[[1]],
                          results_exp_5_h2_color_valence_contingency_memory_excluded$cohens_d$estimate[[1]],
                          results_exp_7_h2_color_valence_contingency_memory_excluded$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h2_color_valence_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_2_h2_color_valence_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_3_h2_color_valence_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_4_h2_color_valence_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_5_h2_color_valence_contingency_memory_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_7_h2_color_valence_contingency_memory_excluded$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h2_color_valence_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_2_h2_color_valence_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_3_h2_color_valence_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_4_h2_color_valence_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_5_h2_color_valence_contingency_memory_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_7_h2_color_valence_contingency_memory_excluded$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2") %>%
  mutate(exclude = TRUE)

# demand_compliance
results_for_meta_analysis_h2_demand_compliance_excluded <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", "Experiment 6",
                            "Experiment 7",
                            "Experiment 8 (standard)", 
                            "Experiment 8 (random)"), 
             cohens_d = c(results_exp_1_h2_demand_compliance_excluded$cohens_d$estimate[[1]], 
                          results_exp_2_h2_demand_compliance_excluded$cohens_d$estimate[[1]], 
                          results_exp_3_h2_demand_compliance_excluded$cohens_d$estimate[[1]], 
                          results_exp_4_h2_demand_compliance_excluded$cohens_d$estimate[[1]],
                          results_exp_5_h2_demand_compliance_excluded$cohens_d$estimate[[1]],
                          results_exp_6_h2_demand_compliance_excluded$cohens_d$estimate[[1]],
                          results_exp_7_h2_demand_compliance_excluded$cohens_d$estimate[[1]],
                          results_exp_8nri_h2_demand_compliance_excluded$cohens_d$estimate[[1]], 
                          results_exp_8ri_h2_demand_compliance_excluded$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h2_demand_compliance_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_2_h2_demand_compliance_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_3_h2_demand_compliance_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_4_h2_demand_compliance_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_5_h2_demand_compliance_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_6_h2_demand_compliance_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_7_h2_demand_compliance_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_8nri_h2_demand_compliance_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_8ri_h2_demand_compliance_excluded$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h2_demand_compliance_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_2_h2_demand_compliance_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_3_h2_demand_compliance_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_4_h2_demand_compliance_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_5_h2_demand_compliance_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_6_h2_demand_compliance_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_7_h2_demand_compliance_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_8nri_h2_demand_compliance_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_8ri_h2_demand_compliance_excluded$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2") %>%
  mutate(exclude = TRUE)

# reactance_self_reports
results_for_meta_analysis_h2_reactance_self_reports_excluded <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", 
                            "Experiment 4", "Experiment 5", "Experiment 6",
                            "Experiment 7",
                            "Experiment 8 (standard)", 
                            "Experiment 8 (random)"), 
             cohens_d = c(results_exp_1_h2_reactance_excluded$cohens_d$estimate[[1]], 
                          results_exp_2_h2_reactance_self_reports_excluded$cohens_d$estimate[[1]], 
                          results_exp_3_h2_reactance_self_reports_excluded$cohens_d$estimate[[1]], 
                          results_exp_4_h2_reactance_self_reports_excluded$cohens_d$estimate[[1]],
                          results_exp_5_h2_reactance_self_reports_excluded$cohens_d$estimate[[1]],
                          results_exp_6_h2_reactance_self_reports_excluded$cohens_d$estimate[[1]],
                          results_exp_7_h2_reactance_self_reports_excluded$cohens_d$estimate[[1]],
                          results_exp_8nri_h2_reactance_self_reports_excluded$cohens_d$estimate[[1]], 
                          results_exp_8ri_h2_reactance_self_reports_excluded$cohens_d$estimate[[1]]),
             ci_lwr = c(results_exp_1_h2_reactance_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_2_h2_reactance_self_reports_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_3_h2_reactance_self_reports_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_4_h2_reactance_self_reports_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_5_h2_reactance_self_reports_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_6_h2_reactance_self_reports_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_7_h2_reactance_self_reports_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_8nri_h2_reactance_self_reports_excluded$cohens_d$conf.int[["lower"]],
                        results_exp_8ri_h2_reactance_self_reports_excluded$cohens_d$conf.int[["lower"]]),
             ci_upr = c(results_exp_1_h2_reactance_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_2_h2_reactance_self_reports_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_3_h2_reactance_self_reports_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_4_h2_reactance_self_reports_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_5_h2_reactance_self_reports_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_6_h2_reactance_self_reports_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_7_h2_reactance_self_reports_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_8nri_h2_reactance_self_reports_excluded$cohens_d$conf.int[["upper"]],
                        results_exp_8ri_h2_reactance_self_reports_excluded$cohens_d$conf.int[["upper"]])) %>%
  mutate(yi = cohens_d, 
         sei = (ci_upr - ci_lwr)/(1.96*2)) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2") %>%
  mutate(exclude = TRUE)

# fit metas
mod_meta_results_h2_us_contingency_memory <- 
  multilevel_moderator_meta_analysis_workflow(data = bind_rows(semi_join(mutate(results_for_meta_analysis_h2_us_contingency_memory, exclude = FALSE),
                                                              results_for_meta_analysis_h2_us_contingency_memory_excluded,
                                                              by = "Experiment"), 
                                                    results_for_meta_analysis_h2_us_contingency_memory_excluded),
                                   effect_size_label = "Cohen's d")

mod_meta_results_h2_color_valence_contingency_memory <- 
  multilevel_moderator_meta_analysis_workflow(data = bind_rows(semi_join(mutate(results_for_meta_analysis_h2_color_valence_contingency_memory, exclude = FALSE),
                                                              results_for_meta_analysis_h2_color_valence_contingency_memory_excluded,
                                                              by = "Experiment"),
                                                    results_for_meta_analysis_h2_color_valence_contingency_memory_excluded),
                                   effect_size_label = "Cohen's d")

mod_meta_results_h2_demand_compliance <- 
  multilevel_moderator_meta_analysis_workflow(data = bind_rows(semi_join(mutate(results_for_meta_analysis_h2_demand_compliance, exclude = FALSE),
                                                              results_for_meta_analysis_h2_demand_compliance_excluded,
                                                              by = "Experiment"),
                                                    results_for_meta_analysis_h2_demand_compliance_excluded), 
                                   effect_size_label = "Cohen's d")

mod_meta_results_h2_reactance <- 
  multilevel_moderator_meta_analysis_workflow(data = bind_rows(semi_join(mutate(results_for_meta_analysis_h2_reactance_self_reports, exclude = FALSE),
                                                              results_for_meta_analysis_h2_reactance_self_reports_excluded,
                                                              by = "Experiment"),
                                                    results_for_meta_analysis_h2_reactance_self_reports_excluded), 
                                   effect_size_label = "Cohen's d")

# forest plots
forest(mod_meta_results_h2_us_contingency_memory$fitted_model)

forest(mod_meta_results_h2_color_valence_contingency_memory$fitted_model)

forest(mod_meta_results_h2_demand_compliance$fitted_model)

forest(mod_meta_results_h2_reactance$fitted_model)

Moderation by:

• US contingency memory (TRUE were excluded): QM(df = 1) = 0.36, p = 0.546, difference in excluded subset Cohen’s d = 0.1, 95% CI [-0.22, 0.41]
• color-valence contingency memory (TRUE were excluded): QM(df = 1) = 5.98, p = 0.014, difference in excluded subset Cohen’s d = -1.46, 95% CI [-2.63, -0.29]
• demand compliance (FALSE were excluded): QM(df = 1) = 16.34, p < .001, difference in excluded subset Cohen’s d = 0.76, 95% CI [0.39, 1.12]
• reactance (FALSE were excluded): QM(df = 1) = 2.44, p = 0.118, difference in excluded subset Cohen’s d = -0.27, 95% CI [-0.61, 0.07]

H3: Behavioural intentions

# re run tests with desired subsets of participants
# exp 1
results_exp_1_h3_us_contingency_memory_excluded <- data_exp_1 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_1_h3_color_valence_contingency_memory_excluded <- data_exp_1 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

# exp 2
results_exp_2_h3_us_contingency_memory_excluded <- data_exp_2 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_2_h3_color_valence_contingency_memory_excluded <- data_exp_2 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

# exp 3
results_exp_3_h3_us_contingency_memory_excluded <- data_exp_3 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_3_h3_color_valence_contingency_memory_excluded <- data_exp_3 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

# exp 4
results_exp_4_h3_us_contingency_memory_excluded <- data_exp_4 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_4_h3_color_valence_contingency_memory_excluded <- data_exp_4 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

# exp 5
results_exp_5_h3_us_contingency_memory_excluded <- data_exp_5 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_5_h3_color_valence_contingency_memory_excluded <- data_exp_5 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

# exp 7
results_exp_7_h3_us_contingency_memory_excluded <- data_exp_7 %>%
  filter(US_contingency_memory == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

results_exp_7_h3_color_valence_contingency_memory_excluded <- data_exp_7 %>%
  filter(color_valence_contingency_memory == FALSE) %>%
  mutate(DV = intention,
         IV = source_valence_condition) %>%
  multinominal_analysis()

# us contingency memory
results_for_meta_analysis_h3_us_contingency_memory_excluded <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", "Experiment 4", 
                            "Experiment 5", "Experiment 7"),
             logOR = c(results_exp_1_h3_us_contingency_memory_excluded$logOR[results_exp_1_h3_us_contingency_memory_excluded$var == "Struan"], 
                       results_exp_2_h3_us_contingency_memory_excluded$logOR[results_exp_2_h3_us_contingency_memory_excluded$var == "Struan"],
                       results_exp_3_h3_us_contingency_memory_excluded$logOR[results_exp_3_h3_us_contingency_memory_excluded$var == "Struan"],
                       results_exp_4_h3_us_contingency_memory_excluded$logOR[results_exp_4_h3_us_contingency_memory_excluded$var == "Struan"],
                       results_exp_5_h3_us_contingency_memory_excluded$logOR[results_exp_5_h3_us_contingency_memory_excluded$var == "Struan"],
                       results_exp_7_h3_us_contingency_memory_excluded$logOR[results_exp_7_h3_us_contingency_memory_excluded$var == "Struan"]),
             logOR_se = c(results_exp_1_h3_us_contingency_memory_excluded$logOR_se[results_exp_1_h3_us_contingency_memory_excluded$var == "Struan"],
                          results_exp_2_h3_us_contingency_memory_excluded$logOR_se[results_exp_2_h3_us_contingency_memory_excluded$var == "Struan"],
                          results_exp_3_h3_us_contingency_memory_excluded$logOR_se[results_exp_3_h3_us_contingency_memory_excluded$var == "Struan"],
                          results_exp_4_h3_us_contingency_memory_excluded$logOR_se[results_exp_4_h3_us_contingency_memory_excluded$var == "Struan"],
                          results_exp_5_h3_us_contingency_memory_excluded$logOR_se[results_exp_5_h3_us_contingency_memory_excluded$var == "Struan"],
                          results_exp_7_h3_us_contingency_memory_excluded$logOR_se[results_exp_7_h3_us_contingency_memory_excluded$var == "Struan"])) %>%
  mutate(yi  = logOR,
         sei = logOR_se) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2") %>%
  mutate(exclude = TRUE)

# color valence contingency memory
results_for_meta_analysis_h3_color_valence_contingency_memory_excluded <- 
  data.frame(Experiment = c("Experiment 1", "Experiment 2", "Experiment 3", "Experiment 4", 
                            "Experiment 5", "Experiment 7"),
             logOR = c(results_exp_1_h3_color_valence_contingency_memory_excluded$logOR[results_exp_1_h3_color_valence_contingency_memory_excluded$var == "Struan"], 
                       results_exp_2_h3_color_valence_contingency_memory_excluded$logOR[results_exp_2_h3_color_valence_contingency_memory_excluded$var == "Struan"],
                       results_exp_3_h3_color_valence_contingency_memory_excluded$logOR[results_exp_3_h3_color_valence_contingency_memory_excluded$var == "Struan"],
                       results_exp_4_h3_color_valence_contingency_memory_excluded$logOR[results_exp_4_h3_color_valence_contingency_memory_excluded$var == "Struan"],
                       results_exp_5_h3_color_valence_contingency_memory_excluded$logOR[results_exp_5_h3_color_valence_contingency_memory_excluded$var == "Struan"],
                       results_exp_7_h3_color_valence_contingency_memory_excluded$logOR[results_exp_7_h3_color_valence_contingency_memory_excluded$var == "Struan"]),
             logOR_se = c(results_exp_1_h3_color_valence_contingency_memory_excluded$logOR_se[results_exp_1_h3_color_valence_contingency_memory$var == "Struan"],
                          results_exp_2_h3_color_valence_contingency_memory_excluded$logOR_se[results_exp_2_h3_color_valence_contingency_memory$var == "Struan"],
                          results_exp_3_h3_color_valence_contingency_memory_excluded$logOR_se[results_exp_3_h3_color_valence_contingency_memory$var == "Struan"],
                          results_exp_4_h3_color_valence_contingency_memory_excluded$logOR_se[results_exp_4_h3_color_valence_contingency_memory$var == "Struan"],
                          results_exp_5_h3_color_valence_contingency_memory_excluded$logOR_se[results_exp_5_h3_color_valence_contingency_memory$var == "Struan"],
                          results_exp_7_h3_color_valence_contingency_memory_excluded$logOR_se[results_exp_7_h3_color_valence_contingency_memory$var == "Struan"])) %>%
  mutate(yi  = logOR,
         sei = logOR_se) %>%
  # remove exp 2 on the basis of the influence test 
  filter(Experiment != "Experiment 2") %>%
  mutate(exclude = TRUE)

# meta fits
mod_meta_results_h3_us_contingency_memory <- 
  multilevel_moderator_meta_analysis_workflow(data = bind_rows(semi_join(mutate(results_for_meta_analysis_h3_us_contingency_memory, exclude = FALSE),
                                                              results_for_meta_analysis_h3_us_contingency_memory_excluded,
                                                              by = "Experiment"),
                                                    results_for_meta_analysis_h3_us_contingency_memory_excluded),
                                   effect_size_label = "Odds Ratio")

mod_meta_results_h3_color_valence_contingency_memory <- 
  multilevel_moderator_meta_analysis_workflow(data = bind_rows(semi_join(mutate(results_for_meta_analysis_h3_color_valence_contingency_memory, exclude = FALSE),
                                                              results_for_meta_analysis_h3_color_valence_contingency_memory_excluded,
                                                              by = "Experiment"),
                                                    results_for_meta_analysis_h3_color_valence_contingency_memory_excluded),
                                   effect_size_label = "Odds Ratio")

# forest plots
forest(mod_meta_results_h3_us_contingency_memory$fitted_model)

forest(mod_meta_results_h3_color_valence_contingency_memory$fitted_model)

Moderation by:

• US contingency memory (TRUE were excluded): QM(df = 1) = 5.29, p = 0.021, difference in excluded subset Odds Ratio = -0.93, 95% CI [-1.72, -0.14]
• color-valence contingency memory (TRUE were excluded): QM(df = 1) = 10.47, p = 0.001, difference in excluded subset Odds Ratio = -2.86, 95% CI [-4.59, -1.13]

ES across moderation analyses

effect_sizes_by_subgroup <- 
  
  bind_rows(
    # IATs
    (data.frame(moderator = c("us_contingency_memory AWARE",
                              "color_valence_contingency_memory AWARE",
                              "demand_NONcompliance",
                              "NONreactant"), 
                es_robustness = c(mod_meta_results_h1_us_contingency_memory$fitted_model$b[1],
                                  mod_meta_results_h1_color_valence_contingency_memory$fitted_model$b[1],
                                  mod_meta_results_h1_demand_compliance$fitted_model$b[1],
                                  mod_meta_results_h1_reactance$fitted_model$b[1]),
                ci_lower_robustness = c(mod_meta_results_h1_us_contingency_memory$fitted_model$ci.lb[1],
                                        mod_meta_results_h1_color_valence_contingency_memory$fitted_model$ci.lb[1],
                                        mod_meta_results_h1_demand_compliance$fitted_model$ci.lb[1],
                                        mod_meta_results_h1_reactance$fitted_model$ci.lb[1]),
                ci_upper_robustness = c(mod_meta_results_h1_us_contingency_memory$fitted_model$ci.ub[1],
                                        mod_meta_results_h1_color_valence_contingency_memory$fitted_model$ci.ub[1],
                                        mod_meta_results_h1_demand_compliance$fitted_model$ci.ub[1],
                                        mod_meta_results_h1_reactance$fitted_model$ci.ub[1]),
                es_excluded = c(sum(mod_meta_results_h1_us_contingency_memory$fitted_model$b),
                                sum(mod_meta_results_h1_color_valence_contingency_memory$fitted_model$b),
                                sum(mod_meta_results_h1_demand_compliance$fitted_model$b),
                                sum(mod_meta_results_h1_reactance$fitted_model$b)),
                ci_lower_excluded = c(sum(mod_meta_results_h1_us_contingency_memory$fitted_model$ci.lb),
                                      sum(mod_meta_results_h1_color_valence_contingency_memory$fitted_model$ci.lb),
                                      sum(mod_meta_results_h1_demand_compliance$fitted_model$ci.lb),
                                      sum(mod_meta_results_h1_reactance$fitted_model$ci.lb)),
                ci_upper_excluded = c(sum(mod_meta_results_h1_us_contingency_memory$fitted_model$ci.ub),
                                      sum(mod_meta_results_h1_color_valence_contingency_memory$fitted_model$ci.ub),
                                      sum(mod_meta_results_h1_demand_compliance$fitted_model$ci.ub),
                                      sum(mod_meta_results_h1_reactance$fitted_model$ci.ub))) %>%
       mutate(DV = "IAT",
              es = "d")),
    
    # Self reported evlautions
    (data.frame(moderator = c("us_contingency_memory AWARE",
                              "color_valence_contingency_memory AWARE",
                              "demand_NONcompliance",
                              "NONreactant"), 
                es_robustness = c(mod_meta_results_h2_us_contingency_memory$fitted_model$b[1],
                                  mod_meta_results_h2_color_valence_contingency_memory$fitted_model$b[1],
                                  mod_meta_results_h2_demand_compliance$fitted_model$b[1],
                                  mod_meta_results_h2_reactance$fitted_model$b[1]),
                ci_lower_robustness = c(mod_meta_results_h2_us_contingency_memory$fitted_model$ci.lb[1],
                                        mod_meta_results_h2_color_valence_contingency_memory$fitted_model$ci.lb[1],
                                        mod_meta_results_h2_demand_compliance$fitted_model$ci.lb[1],
                                        mod_meta_results_h2_reactance$fitted_model$ci.lb[1]),
                ci_upper_robustness = c(mod_meta_results_h2_us_contingency_memory$fitted_model$ci.ub[1],
                                        mod_meta_results_h2_color_valence_contingency_memory$fitted_model$ci.ub[1],
                                        mod_meta_results_h2_demand_compliance$fitted_model$ci.ub[1],
                                        mod_meta_results_h2_reactance$fitted_model$ci.ub[1]),
                es_excluded = c(sum(mod_meta_results_h2_us_contingency_memory$fitted_model$b),
                                sum(mod_meta_results_h2_color_valence_contingency_memory$fitted_model$b),
                                sum(mod_meta_results_h2_demand_compliance$fitted_model$b),
                                sum(mod_meta_results_h2_reactance$fitted_model$b)),
                ci_lower_excluded = c(sum(mod_meta_results_h2_us_contingency_memory$fitted_model$ci.lb),
                                      sum(mod_meta_results_h2_color_valence_contingency_memory$fitted_model$ci.lb),
                                      sum(mod_meta_results_h2_demand_compliance$fitted_model$ci.lb),
                                      sum(mod_meta_results_h2_reactance$fitted_model$ci.lb)),
                ci_upper_excluded = c(sum(mod_meta_results_h2_us_contingency_memory$fitted_model$ci.ub),
                                      sum(mod_meta_results_h2_color_valence_contingency_memory$fitted_model$ci.ub),
                                      sum(mod_meta_results_h2_demand_compliance$fitted_model$ci.ub),
                                      sum(mod_meta_results_h2_reactance$fitted_model$ci.ub))) %>%
       mutate(DV = "self reported evaluations",
              es = "d")),
    
    
    # Behavioural intentions
    (data.frame(moderator = c("us_contingency_memory AWARE",
                              "color_valence_contingency_memory AWARE"), 
                es_robustness = c(mod_meta_results_h3_us_contingency_memory$fitted_model$b[1],
                                  mod_meta_results_h3_color_valence_contingency_memory$fitted_model$b[1]),
                ci_lower_robustness = c(mod_meta_results_h3_us_contingency_memory$fitted_model$ci.lb[1],
                                        mod_meta_results_h3_color_valence_contingency_memory$fitted_model$ci.lb[1]),
                ci_upper_robustness = c(mod_meta_results_h3_us_contingency_memory$fitted_model$ci.ub[1],
                                        mod_meta_results_h3_color_valence_contingency_memory$fitted_model$ci.ub[1]),
                es_excluded = c(sum(mod_meta_results_h3_us_contingency_memory$fitted_model$b),
                                sum(mod_meta_results_h3_color_valence_contingency_memory$fitted_model$b)),
                ci_lower_excluded = c(sum(mod_meta_results_h3_us_contingency_memory$fitted_model$ci.lb),
                                      sum(mod_meta_results_h3_color_valence_contingency_memory$fitted_model$ci.lb)),
                ci_upper_excluded = c(sum(mod_meta_results_h3_us_contingency_memory$fitted_model$ci.ub),
                                      sum(mod_meta_results_h3_color_valence_contingency_memory$fitted_model$ci.ub))) %>%
       mutate(DV = "behvioural intentions",
              es = "OR"))
    
  ) %>%
  round_df(2) %>%
  select(DV, 
         moderator,
         es, 
         es_robustness,
         ci_lower_robustness,
         ci_upper_robustness,
         es_excluded,
         ci_lower_excluded,
         ci_upper_excluded)

effect_sizes_by_subgroup %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

DV	moderator	es	es_robustness	ci_lower_robustness	ci_upper_robustness	es_excluded	ci_lower_excluded	ci_upper_excluded
IAT	us_contingency_memory AWARE	d	0.83	0.54	1.12	0.85	0.15	1.54
IAT	color_valence_contingency_memory AWARE	d	1.11	0.46	1.77	0.33	-1.54	2.20
IAT	demand_NONcompliance	d	0.80	0.46	1.13	0.76	-0.16	1.69
IAT	NONreactant	d	0.85	0.67	1.04	0.83	0.13	1.52
self reported evaluations	us_contingency_memory AWARE	d	1.19	1.02	1.37	1.29	0.80	1.77
self reported evaluations	color_valence_contingency_memory AWARE	d	1.80	1.21	2.40	0.34	-1.43	2.11
self reported evaluations	demand_NONcompliance	d	1.05	0.79	1.30	1.80	1.18	2.43
self reported evaluations	NONreactant	d	1.19	0.94	1.44	0.92	0.33	1.51
behvioural intentions	us_contingency_memory AWARE	OR	1.89	1.47	2.32	0.96	-0.25	2.18
behvioural intentions	color_valence_contingency_memory AWARE	OR	2.88	1.95	3.82	0.03	-2.64	2.69

Sample sizes for moderation analyses

combined_data <- bind_rows(mutate(data_exp_1, experiment = "1"), 
                           #mutate(data_exp_2, experiment = "2"),  # excluded by robustness tests
                           mutate(data_exp_3, experiment = "3"), 
                           mutate(data_exp_4, experiment = "4"), 
                           mutate(data_exp_5, experiment = "5"), 
                           mutate(data_exp_6, experiment = "6"), 
                           mutate(data_exp_7, experiment = "7"), 
                           mutate(data_exp_8ri, experiment = "8ri"), 
                           mutate(data_exp_8nri, experiment = "8nri"))

# n per cells
n_per_cell <- 
  
  bind_rows(
    
    # IATs
    
    # US_contingency_memory
    (combined_data %>%
       # EXCLUDE CONDITIONS THAT HAD INADEQUATE N TO CALCULATE ES IN METAS ABOVE
       filter((!experiment %in% c("8nri") & US_contingency_memory == TRUE) |
                US_contingency_memory == FALSE) %>%  
       select(IAT_Score, source_valence_condition, US_contingency_memory) %>%
       drop_na() %>%
       count(US_contingency_memory) %>%
       mutate(moderator = "US_contingency_memory",
              DV = "IAT") %>%
       spread(US_contingency_memory, n) %>%
       rename(n_robustness_subset = `TRUE`,
              n_excluded_subset = `FALSE`)),
    
    # color_valence_contingency_memory
    (combined_data %>%
       select(IAT_Score, source_valence_condition, color_valence_contingency_memory) %>%
       drop_na() %>%
       count(color_valence_contingency_memory) %>%
       mutate(moderator = "color_valence_contingency_memory",
              DV = "IAT") %>%
       spread(color_valence_contingency_memory, n) %>%
       rename(n_robustness_subset = `TRUE`,
              n_excluded_subset = `FALSE`)),
    
    # demand_IAT
    (combined_data %>%
       # EXCLUDE CONDITIONS THAT HAD INADEQUATE N TO CALCULATE ES IN METAS ABOVE
       filter((!experiment %in% c("6", "8ri", "8nri") & demand_IAT == TRUE) |
                demand_IAT == FALSE) %>%  
       select(IAT_Score, source_valence_condition, demand_IAT) %>%
       drop_na() %>%
       count(demand_IAT) %>%
       mutate(moderator = "demand",
              DV = "IAT") %>%
       spread(demand_IAT, n) %>%
       rename(n_robustness_subset = `FALSE`,
              n_excluded_subset = `TRUE`)),
    
    # reactance_IAT
    (combined_data %>%
       select(IAT_Score, source_valence_condition, reactance_IAT) %>%
       drop_na() %>%
       count(reactance_IAT) %>%
       mutate(moderator = "reactance",
              DV = "IAT") %>%
       spread(reactance_IAT, n) %>%
       rename(n_robustness_subset = `FALSE`,
              n_excluded_subset = `TRUE`)),
    
    
    
    # Self reported evaluations
    
    # US_contingency_memory
    (combined_data %>%
       # EXCLUDE CONDITIONS THAT HAD INADEQUATE N TO CALCULATE ES IN METAS ABOVE
       filter((!experiment %in% c("8nri") & US_contingency_memory == TRUE) |
                US_contingency_memory == FALSE) %>%  
       select(EC_Effect_CS1_CS2_Difference_Score, source_valence_condition, US_contingency_memory) %>%
       drop_na() %>%
       count(US_contingency_memory) %>%
       mutate(moderator = "US_contingency_memory",
              DV = "self reported evaluations") %>%
       spread(US_contingency_memory, n) %>%
       rename(n_robustness_subset = `TRUE`,
              n_excluded_subset = `FALSE`)),
    
    # color_valence_contingency_memory
    (combined_data %>%
       select(EC_Effect_CS1_CS2_Difference_Score, source_valence_condition, color_valence_contingency_memory) %>%
       drop_na() %>%
       count(color_valence_contingency_memory) %>%
       mutate(moderator = "color_valence_contingency_memory",
              DV = "self reported evaluations") %>%
       spread(color_valence_contingency_memory, n) %>%
       rename(n_robustness_subset = `TRUE`,
              n_excluded_subset = `FALSE`)),
    
    # demand_self_reports
    (combined_data %>%
       select(EC_Effect_CS1_CS2_Difference_Score, source_valence_condition, demand_self_reports) %>%
       drop_na() %>%
       count(demand_self_reports) %>%
       mutate(moderator = "demand",
              DV = "self reported evaluations") %>%
       spread(demand_self_reports, n) %>%
       rename(n_robustness_subset = `FALSE`,
              n_excluded_subset = `TRUE`)),
    
    # reactance_self_reports
    (combined_data %>%
       select(EC_Effect_CS1_CS2_Difference_Score, source_valence_condition, reactance_self_reports) %>%
       drop_na() %>%
       count(reactance_self_reports) %>%
       mutate(moderator = "reactance",
              DV = "self reported evaluations") %>%
       spread(reactance_self_reports, n) %>%
       rename(n_robustness_subset = `FALSE`,
              n_excluded_subset = `TRUE`)),
    
    
    # behavioural intentions
    
    # US_contingency_memory
    (combined_data %>%
       select(intention, source_valence_condition, US_contingency_memory) %>%
       drop_na() %>%
       count(US_contingency_memory) %>%
       mutate(moderator = "US_contingency_memory",
              DV = "behavioural intentions") %>%
       spread(US_contingency_memory, n) %>%
       rename(n_robustness_subset = `TRUE`,
              n_excluded_subset = `FALSE`)),
    
    # color_valence_contingency_memory
    (combined_data %>%
       select(intention, source_valence_condition, color_valence_contingency_memory) %>%
       drop_na() %>%
       count(color_valence_contingency_memory) %>%
       mutate(moderator = "color_valence_contingency_memory",
              DV = "behavioural intentions") %>%
       spread(color_valence_contingency_memory, n) %>%
       rename(n_robustness_subset = `TRUE`,
              n_excluded_subset = `FALSE`))
    
  ) %>%
  select(DV, moderator, n_robustness_subset, n_excluded_subset)

# print
n_per_cell %>%
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)

DV	moderator	n_robustness_subset	n_excluded_subset
IAT	US_contingency_memory	484	149
IAT	color_valence_contingency_memory	348	218
IAT	demand	854	75
IAT	reactance	839	96
self reported evaluations	US_contingency_memory	865	256
self reported evaluations	color_valence_contingency_memory	625	429
self reported evaluations	demand	1211	212
self reported evaluations	reactance	1222	201
behavioural intentions	US_contingency_memory	805	249
behavioural intentions	color_valence_contingency_memory	625	429